Repo created

tools/python/airmaps/instruments/settings.py

@@ -0,0 +1,62 @@
+import sys
+from typing import AnyStr
+
+from maps_generator.generator import settings
+
+STORAGE_PREFIX = ""
+
+# Storage settings
+WD_HOST = ""
+WD_LOGIN = ""
+WD_PASSWORD = ""
+
+# Common section
+EMAILS = []
+
+settings.LOGGING["loggers"]["airmaps"] = {
+    "handlers": ["stdout", "file"],
+    "level": "DEBUG",
+    "propagate": True,
+}
+
+
+def get_airmaps_emails(emails: AnyStr):
+    if not emails:
+        return []
+
+    for ch in [",", ";", ":"]:
+        emails.replace(ch, " ")
+
+    return list(filter(None, [e.strpip() for e in emails.join(" ")]))
+
+
+def init(default_settings_path: AnyStr):
+    settings.init(default_settings_path)
+
+    # Try to read a config and to overload default settings
+    cfg = settings.CfgReader(default_settings_path)
+
+    # Storage section
+    global WD_HOST
+    global WD_LOGIN
+    global WD_PASSWORD
+
+    WD_HOST = cfg.get_opt("Storage", "WD_HOST", WD_HOST)
+    WD_LOGIN = cfg.get_opt("Storage", "WD_LOGIN", WD_LOGIN)
+    WD_PASSWORD = cfg.get_opt("Storage", "WD_PASSWORD", WD_PASSWORD)
+
+    # Common section
+    global EMAILS
+    emails = cfg.get_opt("Common", "EMAILS", "")
+    EMAILS = get_airmaps_emails(emails)
+
+    # Import all contains from maps_generator.generator.settings.
+    thismodule = sys.modules[__name__]
+    for name in dir(settings):
+        if not name.startswith("_") and name.isupper():
+            value = getattr(settings, name)
+            setattr(thismodule, name, value)
+
+    global STORAGE_PREFIX
+    if settings.DEBUG:
+        STORAGE_PREFIX = "/tests"

tools/python/airmaps/instruments/storage.py

@@ -0,0 +1,27 @@
+import logging
+
+import webdav.client as wc
+
+from airmaps.instruments import settings
+
+logger = logging.getLogger("airmaps")
+
+WD_OPTIONS = {
+    "webdav_hostname": settings.WD_HOST,
+    "webdav_login": settings.WD_LOGIN,
+    "webdav_password": settings.WD_PASSWORD,
+}
+
+
+def wd_fetch(src, dst):
+    logger.info(f"Fetch form {src} to {dst} with options {WD_OPTIONS}.")
+    client = wc.Client(WD_OPTIONS)
+    client.download_sync(src, dst)
+
+
+def wd_publish(src, dst):
+    logger.info(f"Publish form {src} to {dst} with options {WD_OPTIONS}.")
+    client = wc.Client(WD_OPTIONS)
+    tmp = f"{dst[:-1]}__/" if dst[-1] == "/" else f"{dst}__"
+    client.upload_sync(local_path=src, remote_path=tmp)
+    client.move(remote_path_from=tmp, remote_path_to=dst)

tools/python/airmaps/instruments/utils.py

@@ -0,0 +1,48 @@
+import os
+import shutil
+from datetime import datetime
+from typing import Iterable
+
+from airflow.operators.python_operator import PythonOperator
+
+from maps_generator.generator.env import Env
+from maps_generator.generator.stages import Stage
+from maps_generator.generator.stages import get_stage_name
+from maps_generator.maps_generator import run_generation
+
+
+def put_current_date_in_filename(filename):
+    path, name = os.path.split(filename)
+    parts = name.split(".", maxsplit=1)
+    parts[0] += f"__{datetime.today().strftime('%Y_%m_%d')}"
+    return os.path.join(path, ".".join(parts))
+
+
+def get_latest_filename(filename, prefix=""):
+    path, name = os.path.split(filename)
+    parts = name.split(".", maxsplit=1)
+    assert len(parts) != 0, parts
+    parts[0] = f"{prefix}latest"
+    return os.path.join(path, ".".join(parts))
+
+
+def rm_build(**kwargs):
+    build_name = kwargs["ti"].xcom_pull(key="build_name")
+    env = Env(build_name=build_name)
+    shutil.rmtree(env.build_path)
+
+
+def make_rm_build_task(dag):
+    return PythonOperator(
+        task_id="Rm_build_task",
+        provide_context=True,
+        python_callable=rm_build,
+        dag=dag,
+    )
+
+
+def run_generation_from_first_stage(
+    env: Env, stages: Iterable[Stage], build_lock: bool = True
+):
+    from_stage = get_stage_name(next(iter(stages)))
+    run_generation(env, stages, from_stage, build_lock)