While building one of my SaaS products, I found myself in need of a list of all countries and their cities, and the already existing 3rd-party packages are duplicated. This is why I did my own thing: tinkering. Basically, I found this API by Postman that provides you with different endpoints related to geolocations. You can check it out yourself. HERE.
pip install requests
import requests
# this call gets the list of all countries
url = "https://countriesnow.space/api/v0.1/countries/flag/unicode"
payload = {}
headers = {}
response = requests.request("GET", url, headers=headers, data=payload)
print(response.text)
import requests
# this call gets the list of all cities in a country
url = "https://countriesnow.space/api/v0.1/countries/cities"
payload = 'country=Sudan'
headers = {}
response = requests.request("POST", url, headers=headers, data=payload)
print(response.text)
Now, how can I approach this? I thought of a command that would automate the process of adding all countries and their cities to my database. First, create an app in your project.
python manage.py startapp geolocation
add the new app to your INSTALLED_APPS
INSTALLED_APPS = [
...
'geolocation',
...
]
wWe create the new models:
#models.py
from django.db import models
class Country(models.Model):
name = models.CharField(max_length=100)
iso3 = models.CharField(max_length=3, unique=True)
unicode_flag = models.CharField(max_length=10, null=True, blank=True)
def __str__(self):
return self.name
class City(models.Model):
name = models.CharField(max_length=100)
country = models.ForeignKey(Country, on_delete=models.CASCADE)
def __str__(self):
return f"{self.name}, {self.country.name}"
we apply the new changes to our database
python manage.py makemigrations
python manage.py migrate
Before populating data into our models, I noticed that the previous endpoint that retrieves all countries returns 251 countries. To be neutral, I kept only the UN-recognized countries from this website
Scrapping time:
const elements = document.querySelectorAll('div.col-md-12 > div > div > h2');
elements.forEach((element) => {
console.log(element.textContent);
});
Now, copy the output to a VS Code file and remove any unnecessary strings. To make it usable for a Python script, either read it as a file or put everything into a Python list. How: on VS Code, click Ctrl + F, then activate regex.
Click on Replace All, and you can wrap everything with square brackets, and you've got yourself a list of countries recognized by the UN. For the final part, which is the most important—making this process autonomous—do the following:
Make a folder in your app's folder named management with an empty Python file init.py. Then, create another folder named commands inside the management folder, with an empty init.py file as well. Finally, add our script, which we’re going to name load_countries_and_cities.py.
small tweak
I had to edit/comment on some countries that I scrapped from the UN list. website because they don't exist on the chosen api endpoint
# your_app_name/management/commands/load_countries_and_cities.py
import requests
from django.core.management.base import BaseCommand
from geolocation.models import Country, City
from django.db import transaction
import time
UN_COUNTRIES = [
"Afghanistan",
"Albania",
"Algeria",
"Andorra",
"Angola",
"Antigua and Barbuda",
"Argentina",
"Armenia",
"Australia",
"Austria",
"Azerbaijan",
"Bahamas",
"Bahrain",
"Bangladesh",
"Barbados",
"Belarus",
"Belgium",
"Belize",
"Benin",
"Bhutan",
"Bolivia",
"Bosnia and Herzegovina",
"Botswana",
"Brazil",
"Brunei",
"Bulgaria",
"Burkina Faso",
"Burundi",
"Cape Verde",
"Cambodia",
"Cameroon",
"Canada",
"Central African Republic",
"Chad",
"Chile",
"China",
"Colombia",
"Comoros",
"Congo",
"Costa Rica",
"Ivory Coast",
"Croatia",
"Cuba",
"Cyprus",
# "Czechia",
"North Korea",
# "Democratic Republic of the Congo",
"Denmark",
"Djibouti",
"Dominica",
"Dominican Republic",
"Ecuador",
"Egypt",
"El Salvador",
"Equatorial Guinea",
"Eritrea",
"Estonia",
# "Eswatini",
"Ethiopia",
"Fiji",
"Finland",
"France",
"Gabon",
"Gambia",
"Georgia",
"Germany",
"Ghana",
"Greece",
"Grenada",
"Guatemala",
"Guinea",
"Guinea-Bissau",
"Guyana",
"Haiti",
"Honduras",
"Hungary",
"Iceland",
"India",
"Indonesia",
"Iran",
"Iraq",
"Ireland",
"Israel",
"Italy",
"Jamaica",
"Japan",
"Jordan",
"Kazakhstan",
"Kenya",
"Kiribati",
"Kuwait",
"Kyrgyzstan",
"Laos",
"Latvia",
"Lebanon",
"Lesotho",
"Liberia",
"Libya",
"Liechtenstein",
"Lithuania",
"Luxembourg",
"Madagascar",
"Malawi",
"Malaysia",
"Maldives",
"Mali",
"Malta",
"Marshall Islands",
"Mauritania",
"Mauritius",
"Mexico",
# "Micronesia",
"Monaco",
"Mongolia",
"Montenegro",
"Morocco",
"Mozambique",
"Myanmar",
"Namibia",
"Nauru",
"Nepal",
"Netherlands",
"New Zealand",
"Nicaragua",
"Niger",
"Nigeria",
"Macedonia",
"Norway",
"Oman",
"Pakistan",
"Palau",
"Panama",
"Papua New Guinea",
"Paraguay",
"Peru",
"Philippines",
"Poland",
"Portugal",
"Qatar",
"South Korea",
"Moldova",
"Romania",
"Russia",
"Rwanda",
"Saint Kitts and Nevis",
"Saint Lucia",
"Saint Vincent and the Grenadines",
"Samoa",
"San Marino",
"Sao Tome and Principe",
"Saudi Arabia",
"Senegal",
"Serbia",
"Seychelles",
"Sierra Leone",
"Singapore",
"Slovakia",
"Slovenia",
"Solomon Islands",
"Somalia",
"South Africa",
# "South Sudan",
"Spain",
"Sri Lanka",
"Sudan",
"Suriname",
"Sweden",
"Switzerland",
"Syria",
# "Tajikistan",
"Thailand",
"Timor-Leste",
"Togo",
"Tonga",
"Trinidad and Tobago",
"Tunisia",
"Turkey",
"Turkmenistan",
# "Tuvalu",
"Uganda",
"Ukraine",
"United Arab Emirates",
"United Kingdom",
"Ireland",
"Tanzania",
"United States",
"Uruguay",
"Uzbekistan",
"Vanuatu",
"Venezuela",
"VietNam",
"Yemen",
"Zambia",
"Zimbabwe",
]
class Command(BaseCommand):
help = 'Load all countries their related cities from an external API'
def handle(self, *args, **options):
countries_url = 'https://countriesnow.space/api/v0.1/countries/flag/unicode'
cities_url = 'https://countriesnow.space/api/v0.1/countries/cities'
rate_limit_delay = 0.5
self.stdout.write('start fetching countries...')
try:
response = requests.get(countries_url)
response.raise_for_status()
raw_countries_data = response.json()
raw_countries = raw_countries_data["data"]
filteredCountries = [item for item in raw_countries if item["name"] in UN_COUNTRIES]
countries_data = filteredCountries
except requests.RequestException as e:
self.stderr.write(f'Error fetching countries: {e}')
return
total_countries = len(countries_data)
self.stdout.write(f'Total countries fetched: {total_countries}')
with transaction.atomic():
for idx, country_data in enumerate(countries_data, start=1):
country_name = country_data.get('name')
iso3 = country_data.get('iso3')
unicode_flag = country_data.get('unicodeFlag')
if not country_name or not iso3:
self.stderr.write(f'Skipping invalid country data: {country_data}')
continue
country, created = Country.objects.get_or_create(
iso3=iso3,
defaults={'name': country_name, 'unicode_flag': unicode_flag}
)
if not created:
# Update name and unicode_flag if necessary
updated = False
if country.name != country_name:
country.name = country_name
updated = True
if country.unicode_flag != unicode_flag:
country.unicode_flag = unicode_flag
updated = True
if updated:
country.save()
self.stdout.write(f'Processing country {idx}/{total_countries}: {country.name}')
# Fetch cities for this country
# Since the API requires a POST request with payload {country: countryName}
try:
cities_response = requests.post(
cities_url,
json={'country': country_name}
)
cities_response.raise_for_status()
raw_cities_data = cities_response.json()
cities_data = raw_cities_data["data"]
except requests.RequestException as e:
self.stderr.write(f'Error fetching cities for country {country_name}: {e}')
continue # Skip to next country
if not cities_data:
self.stdout.write(f'No cities found for country {country.name}')
continue
# Optionally, delete existing cities for this country to prevent duplicates
City.objects.filter(country=country).delete()
# Create city objects
city_objects = []
for city_name in cities_data:
city = City(name=city_name, country=country)
city_objects.append(city)
# Bulk create cities
City.objects.bulk_create(city_objects, ignore_conflicts=True)
self.stdout.write(f'Added {len(city_objects)} cities for country {country.name}')
# Delay to avoid hitting rate limits
time.sleep(rate_limit_delay)
self.stdout.write('Data loading complete.')
and now the bew magic command
python manage.py load_countries_and_cities
And voilaaa all what's left is a view according to your needs
from geolocation.models import Country, City
all_countries = Country.objects.all()
all_moroccan_cities = Cities.objects.filter(country__name="Morocco")
Top comments (0)