Repo created
This commit is contained in:
parent
4af19165ec
commit
68073add76
12458 changed files with 12350765 additions and 2 deletions
93
tools/python/airmaps/dags/build_coastline.py
Normal file
93
tools/python/airmaps/dags/build_coastline.py
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
import logging
|
||||
import os
|
||||
import shutil
|
||||
from datetime import timedelta
|
||||
|
||||
from airflow import DAG
|
||||
from airflow.operators.python_operator import PythonOperator
|
||||
from airflow.utils.dates import days_ago
|
||||
|
||||
from airmaps.instruments import settings
|
||||
from airmaps.instruments import storage
|
||||
from airmaps.instruments.utils import get_latest_filename
|
||||
from airmaps.instruments.utils import make_rm_build_task
|
||||
from airmaps.instruments.utils import put_current_date_in_filename
|
||||
from airmaps.instruments.utils import rm_build
|
||||
from maps_generator.generator import stages_declaration as sd
|
||||
from maps_generator.generator.env import Env
|
||||
from maps_generator.generator.env import WORLD_COASTS_NAME
|
||||
from maps_generator.maps_generator import run_generation
|
||||
|
||||
logger = logging.getLogger("airmaps")
|
||||
|
||||
|
||||
DAG = DAG(
|
||||
"Build_coastline",
|
||||
schedule_interval=timedelta(days=1),
|
||||
default_args={
|
||||
"owner": "OMaps",
|
||||
"depends_on_past": True,
|
||||
"start_date": days_ago(0),
|
||||
"email": settings.EMAILS,
|
||||
"email_on_failure": True,
|
||||
"email_on_retry": False,
|
||||
"retries": 0,
|
||||
"retry_delay": timedelta(minutes=5),
|
||||
"priority_weight": 1,
|
||||
},
|
||||
)
|
||||
|
||||
COASTLINE_STORAGE_PATH = f"{settings.STORAGE_PREFIX}/coasts"
|
||||
|
||||
|
||||
def publish_coastline(**kwargs):
|
||||
build_name = kwargs["ti"].xcom_pull(key="build_name")
|
||||
env = Env(build_name=build_name)
|
||||
for name in (f"{WORLD_COASTS_NAME}.geom", f"{WORLD_COASTS_NAME}.rawgeom"):
|
||||
coastline = put_current_date_in_filename(name)
|
||||
latest = get_latest_filename(name)
|
||||
coastline_full = os.path.join(env.paths.coastline_path, coastline)
|
||||
latest_full = os.path.join(env.paths.coastline_path, latest)
|
||||
shutil.move(os.path.join(env.paths.coastline_path, name), coastline_full)
|
||||
os.symlink(coastline, latest_full)
|
||||
|
||||
storage.wd_publish(coastline_full, f"{COASTLINE_STORAGE_PATH}/{coastline}")
|
||||
storage.wd_publish(latest_full, f"{COASTLINE_STORAGE_PATH}/{latest}")
|
||||
|
||||
|
||||
def build_coastline(**kwargs):
|
||||
env = Env()
|
||||
kwargs["ti"].xcom_push(key="build_name", value=env.build_name)
|
||||
|
||||
run_generation(
|
||||
env,
|
||||
(
|
||||
sd.StageDownloadAndConvertPlanet(),
|
||||
sd.StageCoastline(use_old_if_fail=False),
|
||||
sd.StageCleanup(),
|
||||
),
|
||||
)
|
||||
env.finish()
|
||||
|
||||
|
||||
BUILD_COASTLINE_TASK = PythonOperator(
|
||||
task_id="Build_coastline_task",
|
||||
provide_context=True,
|
||||
python_callable=build_coastline,
|
||||
on_failure_callback=lambda ctx: rm_build(**ctx),
|
||||
dag=DAG,
|
||||
)
|
||||
|
||||
|
||||
PUBLISH_COASTLINE_TASK = PythonOperator(
|
||||
task_id="Publish_coastline_task",
|
||||
provide_context=True,
|
||||
python_callable=publish_coastline,
|
||||
dag=DAG,
|
||||
)
|
||||
|
||||
|
||||
RM_BUILD_TASK = make_rm_build_task(DAG)
|
||||
|
||||
|
||||
BUILD_COASTLINE_TASK >> PUBLISH_COASTLINE_TASK >> RM_BUILD_TASK
|
||||
154
tools/python/airmaps/dags/build_maps.py
Normal file
154
tools/python/airmaps/dags/build_maps.py
Normal file
|
|
@ -0,0 +1,154 @@
|
|||
import logging
|
||||
from datetime import timedelta
|
||||
|
||||
from airflow import DAG
|
||||
from airflow.operators.python_operator import PythonOperator
|
||||
from airflow.utils.dates import days_ago
|
||||
|
||||
from airmaps.instruments import settings
|
||||
from airmaps.instruments import storage
|
||||
from airmaps.instruments.utils import make_rm_build_task
|
||||
from airmaps.instruments.utils import run_generation_from_first_stage
|
||||
from maps_generator.generator import stages_declaration as sd
|
||||
from maps_generator.generator.env import Env
|
||||
from maps_generator.generator.env import PathProvider
|
||||
from maps_generator.generator.env import get_all_countries_list
|
||||
from maps_generator.maps_generator import run_generation
|
||||
|
||||
logger = logging.getLogger("airmaps")
|
||||
|
||||
|
||||
MAPS_STORAGE_PATH = f"{settings.STORAGE_PREFIX}/maps"
|
||||
|
||||
|
||||
class MapsGenerationDAG(DAG):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
build_prolog_task = PythonOperator(
|
||||
task_id="Build_prolog_task",
|
||||
provide_context=True,
|
||||
python_callable=MapsGenerationDAG.build_prolog,
|
||||
dag=self,
|
||||
)
|
||||
|
||||
build_epilog_task = PythonOperator(
|
||||
task_id="Build_epilog_task",
|
||||
provide_context=True,
|
||||
python_callable=MapsGenerationDAG.build_epilog,
|
||||
dag=self,
|
||||
)
|
||||
|
||||
publish_maps_task = PythonOperator(
|
||||
task_id="Publish_maps_task",
|
||||
provide_context=True,
|
||||
python_callable=MapsGenerationDAG.publish_maps,
|
||||
dag=self,
|
||||
)
|
||||
|
||||
rm_build_task = make_rm_build_task(self)
|
||||
|
||||
build_epilog_task >> publish_maps_task >> rm_build_task
|
||||
for country in get_all_countries_list(PathProvider.borders_path()):
|
||||
build_prolog_task >> self.make_mwm_operator(country) >> build_epilog_task
|
||||
|
||||
@staticmethod
|
||||
def get_params(namespace="env", **kwargs):
|
||||
return kwargs.get("params", {}).get(namespace, {})
|
||||
|
||||
@staticmethod
|
||||
def build_prolog(**kwargs):
|
||||
params = MapsGenerationDAG.get_params(**kwargs)
|
||||
env = Env(**params)
|
||||
kwargs["ti"].xcom_push(key="build_name", value=env.build_name)
|
||||
run_generation(
|
||||
env,
|
||||
(
|
||||
sd.StageDownloadAndConvertPlanet(),
|
||||
sd.StageCoastline(),
|
||||
sd.StagePreprocess(),
|
||||
sd.StageFeatures(),
|
||||
sd.StageDownloadDescriptions(),
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def make_build_mwm_func(country):
|
||||
def build_mwm(**kwargs):
|
||||
build_name = kwargs["ti"].xcom_pull(key="build_name")
|
||||
params = MapsGenerationDAG.get_params(**kwargs)
|
||||
params.update({"build_name": build_name, "countries": [country,]})
|
||||
env = Env(**params)
|
||||
# We need to check existing of mwm.tmp. It is needed if we want to
|
||||
# build mwms from part of planet.
|
||||
tmp_mwm_name = env.get_tmp_mwm_names()
|
||||
assert len(tmp_mwm_name) <= 1
|
||||
if not tmp_mwm_name:
|
||||
logger.warning(f"mwm.tmp does not exist for {country}.")
|
||||
return
|
||||
|
||||
run_generation_from_first_stage(env, (sd.StageMwm(),), build_lock=False)
|
||||
|
||||
return build_mwm
|
||||
|
||||
@staticmethod
|
||||
def build_epilog(**kwargs):
|
||||
build_name = kwargs["ti"].xcom_pull(key="build_name")
|
||||
params = MapsGenerationDAG.get_params(**kwargs)
|
||||
params.update({"build_name": build_name})
|
||||
env = Env(**params)
|
||||
run_generation_from_first_stage(
|
||||
env,
|
||||
(
|
||||
sd.StageCountriesTxt(),
|
||||
sd.StageLocalAds(),
|
||||
sd.StageStatistics(),
|
||||
sd.StageCleanup(),
|
||||
),
|
||||
)
|
||||
env.finish()
|
||||
|
||||
@staticmethod
|
||||
def publish_maps(**kwargs):
|
||||
build_name = kwargs["ti"].xcom_pull(key="build_name")
|
||||
params = MapsGenerationDAG.get_params(**kwargs)
|
||||
params.update({"build_name": build_name})
|
||||
env = Env(**params)
|
||||
subdir = MapsGenerationDAG.get_params(namespace="storage", **kwargs)["subdir"]
|
||||
storage_path = f"{MAPS_STORAGE_PATH}/{subdir}"
|
||||
storage.wd_publish(env.paths.mwm_path, f"{storage_path}/{env.mwm_version}/")
|
||||
|
||||
def make_mwm_operator(self, country):
|
||||
normalized_name = "__".join(country.lower().split())
|
||||
return PythonOperator(
|
||||
task_id=f"Build_country_{normalized_name}_task",
|
||||
provide_context=True,
|
||||
python_callable=MapsGenerationDAG.make_build_mwm_func(country),
|
||||
dag=self,
|
||||
)
|
||||
|
||||
|
||||
PARAMS = {"storage": {"subdir": "open_source"}}
|
||||
if settings.DEBUG:
|
||||
PARAMS["env"] = {
|
||||
# The planet file in debug mode does not contain Russia_Moscow territory.
|
||||
# It is needed for testing.
|
||||
"countries": ["Cuba", "Haiti", "Jamaica", "Cayman Islands", "Russia_Moscow"]
|
||||
}
|
||||
|
||||
OPEN_SOURCE_MAPS_GENERATION_DAG = MapsGenerationDAG(
|
||||
"Generate_open_source_maps",
|
||||
schedule_interval=timedelta(days=7),
|
||||
default_args={
|
||||
"owner": "OMaps",
|
||||
"depends_on_past": True,
|
||||
"start_date": days_ago(0),
|
||||
"email": settings.EMAILS,
|
||||
"email_on_failure": True,
|
||||
"email_on_retry": False,
|
||||
"retries": 0,
|
||||
"retry_delay": timedelta(minutes=5),
|
||||
"priority_weight": 1,
|
||||
"params": PARAMS,
|
||||
},
|
||||
)
|
||||
83
tools/python/airmaps/dags/update_planet.py
Normal file
83
tools/python/airmaps/dags/update_planet.py
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
import logging
|
||||
from datetime import timedelta
|
||||
|
||||
from airflow import DAG
|
||||
from airflow.operators.python_operator import PythonOperator
|
||||
from airflow.utils.dates import days_ago
|
||||
|
||||
from airmaps.instruments import settings
|
||||
from airmaps.instruments import storage
|
||||
from airmaps.instruments.utils import make_rm_build_task
|
||||
from maps_generator.generator import stages_declaration as sd
|
||||
from maps_generator.generator.env import Env
|
||||
from maps_generator.maps_generator import run_generation
|
||||
from maps_generator.utils.md5 import md5_ext
|
||||
|
||||
logger = logging.getLogger("airmaps")
|
||||
|
||||
|
||||
DAG = DAG(
|
||||
"Update_planet",
|
||||
schedule_interval=timedelta(days=1),
|
||||
default_args={
|
||||
"owner": "OMaps",
|
||||
"depends_on_past": True,
|
||||
"start_date": days_ago(0),
|
||||
"email": settings.EMAILS,
|
||||
"email_on_failure": True,
|
||||
"email_on_retry": False,
|
||||
"retries": 0,
|
||||
"retry_delay": timedelta(minutes=5),
|
||||
"priority_weight": 1,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
PLANET_STORAGE_PATH = f"{settings.STORAGE_PREFIX}/planet_regular/planet-latest.o5m"
|
||||
|
||||
|
||||
def update_planet(**kwargs):
|
||||
env = Env()
|
||||
kwargs["ti"].xcom_push(key="build_name", value=env.build_name)
|
||||
|
||||
if settings.DEBUG:
|
||||
env.add_skipped_stage(sd.StageUpdatePlanet)
|
||||
|
||||
run_generation(
|
||||
env,
|
||||
(
|
||||
sd.StageDownloadAndConvertPlanet(),
|
||||
sd.StageUpdatePlanet(),
|
||||
sd.StageCleanup(),
|
||||
),
|
||||
)
|
||||
env.finish()
|
||||
|
||||
|
||||
def publish_planet(**kwargs):
|
||||
build_name = kwargs["ti"].xcom_pull(key="build_name")
|
||||
env = Env(build_name=build_name)
|
||||
storage.wd_publish(env.paths.planet_o5m, PLANET_STORAGE_PATH)
|
||||
storage.wd_publish(md5_ext(env.paths.planet_o5m), md5_ext(PLANET_STORAGE_PATH))
|
||||
|
||||
|
||||
UPDATE_PLANET_TASK = PythonOperator(
|
||||
task_id="Update_planet_task",
|
||||
provide_context=True,
|
||||
python_callable=update_planet,
|
||||
dag=DAG,
|
||||
)
|
||||
|
||||
|
||||
PUBLISH_PLANET_TASK = PythonOperator(
|
||||
task_id="Publish_planet_task",
|
||||
provide_context=True,
|
||||
python_callable=publish_planet,
|
||||
dag=DAG,
|
||||
)
|
||||
|
||||
|
||||
RM_BUILD_TASK = make_rm_build_task(DAG)
|
||||
|
||||
|
||||
UPDATE_PLANET_TASK >> PUBLISH_PLANET_TASK >> RM_BUILD_TASK
|
||||
Loading…
Add table
Add a link
Reference in a new issue