diff --git a/docs/guides/generating-mosaics.md b/docs/guides/generating-mosaics.md new file mode 100644 index 00000000..448b73f4 --- /dev/null +++ b/docs/guides/generating-mosaics.md @@ -0,0 +1,28 @@ +# Generating mosaics + +Once layers are grouped into layersets, and each layerset is trimmed in the [multimask](./trimming.md) interface, the final step is to collapse the content into a single mosaic per layerset, like finally gluing down all the individual clippings in a collage. You'll find these within within a map's **Mosaic** → **Derivatives** (see [New Iberia, 1885](https://oldinsurancemaps.net/map/sanborn03375_001#mosaic)). + +Currently, OIM supports the creation of two different derivatives per layerset: + +- **Cloud Optimized GeoTIFFs (COG)** + - This is the primary output, and when present, is used within OIM in certain contexts +- **Static XYZ Tileset** + - This is a rendering of the latest COG mosaic to actual PNG tiles in a {z}/{y}/{x}.png folder structure. + +!!! note + + You may be wondering what happens to Key Maps and other categories of maps, i.e. other layersets. Though these are handled on the backend exactly like the Main Content layerset, they are only shown in the derivatives list if they have more than one layer. + +## Keeping derivatives up-to-date + +The creation of a single geotiff (or static XYZ tileset) is effectively a snapshot of the work that has been performed on a map's layers and masks up to that point. Timestamps are displayed showing when masks were last updated, and when derivatives were last generated. + +![up-to-date cog derivative with timestamps](../images/mosaic-cog-fresh.png) + +However, this means that if the ground control points are subsequently updated for a layer, or the masks are adjusted, the derivative artifact will be out-of-date. In this case, the interface will show timestamps of out-of-date artifacts in red and a button will be available allowing you to queue this artifact to be rebuilt. + +![out-of-date cog derivative with red timestamp](../images/mosaic-cog-stale.png) + +Because the process to create a single geotiff (or xyz tileset) from dozens or hundreds of layers can take several hours, we use a queueing system to ensure the server is never overloaded. You can see all mosaicking processes (queued, running, or completed) in [Jobs](https://oldinsurancemaps.net/jobs). + +In the current setup, only two jobs are allowed to run at any time, and queued mosaic operations are processed with a first in, first out policy. diff --git a/docs/guides/services.md b/docs/guides/services.md new file mode 100644 index 00000000..6c8c98b3 --- /dev/null +++ b/docs/guides/services.md @@ -0,0 +1,48 @@ + +# Downloads & web services + +Historical map content can be accessed in a number of different ways and used in many different applications. Throughout the platform you will find places to download or link to: + +1. Individual layers that have been georeferenced + - Find these within a map's **Georeferenced** section: https://oldinsurancemaps.net/map/sanborn03375_001#georeferenced +2. Unified mosaics made from multiple layers + - Find these within a map's **Mosaic** → **Derivatives** section: https://oldinsurancemaps.net/map/sanborn03375_001#mosaic + +!!! warning + + When downloading or otherwise accessing maps, please keep in mind that **OldInsuranceMaps.net provides no guarantee as to the accuracy of georeferenced content.** + +!!! important + + Over time, someone may come along and improve the georeferencing on a layer, or update multimask boundaries and regenerated a mosaic. When this happens the original file is deleted and a new one is created. Because many of the endpoints described below contain the actual filename, **tile service links in 3rd-party integrations may need to be updated after the source layer or mosaic has been updated**. + +## Downloadable geotiffs (COGs) + +OIM relies heavily on [cloud optimized geotiffs](https://cogeo.org/), or "COGs", and you can download these files at any time and easily use them directly in software like QGIS or ArcGIS Desktop. All COGs use JPEG compression in order to keep file sizes manageable. + +## Tile services + +OIM provides service endpoints that can be used in many different applications. The particular service endpoint you use will largely depend on where you are trying to use it, as not all applications support all types of endpoints. + +Generally speaking, XYZ tiles are the most widely supported, TileJSON is a superset spec that wraps an XYZ tile endpoint with some extra information, and WMS is an older standard still in use in many applications. + +OIM uses TiTiler to dynamically produce tiled services from the same source COG files that you can download as described above. + +### XYZ tiles + +XYZ tiles are a very widely supported standard supported by most geospatial software and mapping applications. Here is more info for how to use these URLs in desktop software like [QGIS](https://docs.qgis.org/3.22/en/docs/user_manual/managing_data_source/opening_data.html#using-xyz-tile-services) and [ArcGIS](https://esribelux.com/2021/04/16/xyz-tile-layers-in-arcgis-platform/), as well as web mapping applications like [Leaflet](https://leafletjs.com/reference.html#tilelayer), [OpenLayers](https://openlayers.org/en/latest/examples/xyz.html), [Mapbox/Maplibre GL JS](https://maplibre.org/maplibre-gl-js-docs/example/map-tiles/), [ArcGIS Online](https://doc.arcgis.com/en/arcgis-online/create-maps/add-layers-from-url.htm#ESRI_SECTION1_13A68145A4F94A338AB2002AD5F9844D), and [OpenHistoricalMap (iD Editor)](https://learnosm.org/en/beginner/id-editor/#configuring-the-background-layer). + +!!! note + + Some mosaics will also have an **Static XYZ Tileset** available. This is a static, pre-rendered tileset that you can download, unpack, and host on your own infrastructure. It is different from the XYZ tiles endpoints in that it does not rely on any dynamic tiling service, like TiTiler. + +### TileJSON + +[TileJSON](https://github.com/mapbox/tilejson-spec) is a useful spec that wraps an XYZ endpoint with a few other pieces of information. + +Currently, this is the most reliable OIM-provided endpoint to use because while the underlying XYZ url which may change (as described above), the TileJSON url itself will not change. However, TileJSON is not as widely supported as XYZ tiles. + + +## IIIF Georeference Extension + +For each layer and mosaic you can also access a collection of the georeferencing information formatted as the [IIIF Georeference Extension](https://iiif.io/api/extension/georef/). This annotation will includes the IIIF URLs from the source resources in the Library of Congress and when you open it in Allmaps Viewer, the tiles will be coming directly from LOC and being warped in the browser according to control points and masks created in OIM. Learn more in this slideshow: [creating IIIF georeference annotations](https://docs.google.com/presentation/d/18FRpq1Lcz3T7WincZNRvTjdL87ULy1bU68BCxr425dk/edit?usp=sharing). \ No newline at end of file diff --git a/docs/guides/trimming.md b/docs/guides/trimming.md index 8d9e9643..67620b45 100644 --- a/docs/guides/trimming.md +++ b/docs/guides/trimming.md @@ -2,4 +2,8 @@ Overlapping edges of layers cause messy mosaics, so a mask must be added to each layer to remove its margins. These masks should all be contiguous (sharing adjacent borders and vertices) so it is best to handle them at the map-level, not by masking each layer individually. This is the idea behind a **MultiMask**, which is a unified collection of masks for all layers within a map. -![The multitrim interface.](../images/multitrim.gif) \ No newline at end of file +![The multitrim interface.](../images/multitrim.gif) + +## Masking across multiple volumes + +In large cities it is common to have multiple volumes for a single atlas edition, each volume only covering a subset of the entire city. In the OIM data model, these volumes are treated as separate entities, so while the pages within each one can be carefully masked to form a mosaic, there is no affordance for combining those mosaics across all volumes within an edition. This is something we hope to support in the future. \ No newline at end of file diff --git a/docs/images/mosaic-cog-fresh.png b/docs/images/mosaic-cog-fresh.png new file mode 100644 index 00000000..fe4842bb Binary files /dev/null and b/docs/images/mosaic-cog-fresh.png differ diff --git a/docs/images/mosaic-cog-stale.png b/docs/images/mosaic-cog-stale.png new file mode 100644 index 00000000..ddd87565 Binary files /dev/null and b/docs/images/mosaic-cog-stale.png differ diff --git a/ohmg/api/filters.py b/ohmg/api/filters.py index 22a15c80..8042a7ba 100644 --- a/ohmg/api/filters.py +++ b/ohmg/api/filters.py @@ -17,6 +17,11 @@ class FilterSessionSchema(FilterSchema): type: Optional[str] +class FilterJobSchema(FilterSchema): + operation: Optional[str] = Field(q="operation") + stage: Optional[str] = Field(q="stage") + + class FilterDocumentSchema(FilterSchema): map: str = Field(q="map_id") prepared: Optional[bool] = Field(q="prepared") diff --git a/ohmg/api/routes.py b/ohmg/api/routes.py index db1b8d52..c40a43de 100644 --- a/ohmg/api/routes.py +++ b/ohmg/api/routes.py @@ -21,6 +21,7 @@ Region, ) from ohmg.georeference.models import ( + Job, SessionBase, SessionLock, ) @@ -29,6 +30,7 @@ from .filters import ( FilterAllDocumentsSchema, FilterDocumentSchema, + FilterJobSchema, FilterRegionSchema, FilterSessionSchema, ) @@ -39,6 +41,7 @@ ) from .schemas import ( DocumentSchema, + JobSchema, LayerSchema, LayerSetSchema, MapFullSchema, @@ -76,6 +79,14 @@ def authenticate(self, request, key): ) +@beta2.get("jobs/", response=List[JobSchema], url_name="job_list") +@paginate +def list_jobs(request, filters: FilterJobSchema = Query(...)): + queryset = Job.objects.all().order_by("-date_queued") + queryset = filters.filter(queryset) + return queryset + + @beta2.get("sessions/", response=List[SessionSchema], url_name="session_list") @paginate(SessionPagination) def list_sessions(request, filters: FilterSessionSchema = Query(...), date_range: str = ""): diff --git a/ohmg/api/schemas.py b/ohmg/api/schemas.py index f4b5d916..8a10676a 100644 --- a/ohmg/api/schemas.py +++ b/ohmg/api/schemas.py @@ -29,6 +29,13 @@ logger = logging.getLogger(__name__) +def _datefield_to_timestamp(obj, field: str) -> float | None: + try: + return getattr(obj, field).timestamp() + except (AttributeError, ValueError): + return None + + class UserSchema(Schema): username: str profile_url: str @@ -476,6 +483,7 @@ def resolve_urls(obj): class LayerSetSchema(Schema): + pk: int id: str name: str map_id: str @@ -487,6 +495,9 @@ class LayerSetSchema(Schema): multimask_extent: Optional[tuple] mosaic_cog_url: Optional[str] xyz_tiles_url: Optional[str] + multimask_date: Optional[float] + latest_cog_job: Optional["JobSchema"] + latest_xyz_job: Optional["JobSchema"] @staticmethod def resolve_id(obj): @@ -504,6 +515,18 @@ def resolve_layers_masked_ct(obj): def resolve_name(obj): return str(obj.category) + @staticmethod + def resolve_multimask_date(obj): + return _datefield_to_timestamp(obj, "multimask_date") + + @staticmethod + def resolve_latest_cog_job(obj): + return obj.get_latest_cog_job() + + @staticmethod + def resolve_latest_xyz_job(obj): + return obj.get_latest_xyz_job() + class LayerSetDisplaySchema(Schema): map_id: str @@ -861,6 +884,49 @@ def resolve_status(obj): return "split" +class JobTargetSchema(Schema): + name: str + url: str + + @staticmethod + def resolve_name(obj): + return str(obj) + + @staticmethod + def resolve_url(obj): + return f"/map/{obj.map.pk}" + + +class JobSchema(Schema): + id: int + operation: str + stage: str + message: Optional[str] + target: JobTargetSchema + date_created: Optional[float] + date_queued: Optional[float] + date_started: Optional[float] + date_ended: Optional[float] + run_duration: Optional[int] + + @staticmethod + def resolve_date_created(obj): + return _datefield_to_timestamp(obj, "date_created") + + @staticmethod + def resolve_date_queued(obj): + return _datefield_to_timestamp(obj, "date_queued") + + @staticmethod + def resolve_date_started(obj): + return _datefield_to_timestamp(obj, "date_started") + + @staticmethod + def resolve_date_ended(obj): + return _datefield_to_timestamp(obj, "date_ended") + + DocumentFullSchema.update_forward_refs() RegionFullSchema.update_forward_refs() MapListSchema2.update_forward_refs() +LayerSetSchema.update_forward_refs() diff --git a/ohmg/conf/http.py b/ohmg/conf/http.py index 128d8d30..99db8a2b 100644 --- a/ohmg/conf/http.py +++ b/ohmg/conf/http.py @@ -38,7 +38,7 @@ def user_info_from_request(request): user_info["api_keys"] = user.api_keys user_info["is_authenticated"] = True user_info["is_staff"] = user.is_staff - user_info["perms"] = list(user.get_all_permissions()) + user_info["perms"] = sorted(list(user.get_all_permissions())) else: user_info = { "is_authenticated": False, @@ -90,7 +90,7 @@ def __init__(self, message="object not found"): class JsonResponseUnauthorized(JsonResponse): def __init__(self, message="unauthorized"): - self.status_code = HTTPStatus.NOT_FOUND + self.status_code = HTTPStatus.UNAUTHORIZED super().__init__( { "success": False, @@ -101,7 +101,7 @@ def __init__(self, message="unauthorized"): class JsonResponseBadRequest(JsonResponse): def __init__(self, message="bad request"): - self.status_code = HTTPStatus.NOT_FOUND + self.status_code = HTTPStatus.BAD_REQUEST super().__init__( { "success": False, diff --git a/ohmg/conf/management/commands/configure-services.py b/ohmg/conf/management/commands/configure-services.py index 270da862..ce181973 100644 --- a/ohmg/conf/management/commands/configure-services.py +++ b/ohmg/conf/management/commands/configure-services.py @@ -220,7 +220,7 @@ def generate_celery_mosaic_service(self, state_path: Path): --statedb={str(state_path.resolve())}/celery_mosaic_worker.state \\ --loglevel=INFO \\ --logfile={log_dir}/celery_mosaic.log \\ - --concurrency=1 -n worker2@%h + --concurrency={settings.MAX_CONCURRENT_MOSAIC_JOBS} -n worker2@%h Restart=always [Install] diff --git a/ohmg/conf/settings.py b/ohmg/conf/settings.py index 9b64e0a7..6bae135c 100644 --- a/ohmg/conf/settings.py +++ b/ohmg/conf/settings.py @@ -303,6 +303,8 @@ # this will be removed once Django is upgraded SWAP_COORDINATE_ORDER = ast.literal_eval(os.getenv("SWAP_COORDINATE_ORDER", "False")) +MAX_CONCURRENT_MOSAIC_JOBS = os.getenv("MAX_CONCURRENT_MOSAIC_JOBS", 1) + # CONFIGURE CELERY CELERY_BROKER_URL = os.getenv("BROKER_URL") CELERY_RESULT_BACKEND = "rpc://" @@ -327,6 +329,7 @@ "ohmg.georeference.tasks.delete_stale_sessions": {"queue": "background"}, "ohmg.georeference.tasks.delete_preview_vrts": {"queue": "background"}, "ohmg.georeference.tasks.cleanup_existing_tileset": {"queue": "background"}, + "ohmg.georeference.tasks.run_queued_mosaic_jobs": {"queue": "background"}, "ohmg.georeference.tasks.create_mosaic_cog": {"queue": "mosaic"}, "ohmg.georeference.tasks.create_mosaic_tileset": {"queue": "mosaic"}, } @@ -336,7 +339,11 @@ "remove_stale_sessions": { "task": "ohmg.georeference.tasks.delete_stale_sessions", "schedule": 60.0, - } + }, + "run_queued_mosaic_jobs": { + "task": "ohmg.georeference.tasks.run_queued_mosaic_jobs", + "schedule": 15.0, + }, } # note: this is app_label.ModelClass, diff --git a/ohmg/core/admin.py b/ohmg/core/admin.py index 5ec5a4a5..84b61476 100644 --- a/ohmg/core/admin.py +++ b/ohmg/core/admin.py @@ -61,12 +61,18 @@ class LayerAdmin(admin.ModelAdmin): class LayerSetAdmin(admin.ModelAdmin): + list_display = ( + "map", + "category", + "multimask_date", + ) raw_id_fields = ("map",) readonly_fields = ( "layer_display_list", "extent", "multimask_extent", "xyz_tiles_url", + "multimask_date", ) search_fields = ("map__title",) list_filter = ("category",) diff --git a/ohmg/core/migrations/0014_alter_permissions_options_and_more.py b/ohmg/core/migrations/0014_alter_permissions_options_and_more.py new file mode 100644 index 00000000..e692984f --- /dev/null +++ b/ohmg/core/migrations/0014_alter_permissions_options_and_more.py @@ -0,0 +1,32 @@ +# Generated by Django 4.2.27 on 2026-06-23 09:40 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0013_permissions'), + ] + + operations = [ + migrations.AlterModelOptions( + name='permissions', + options={'default_permissions': (), 'managed': False, 'permissions': (('use_helmert', 'Can use Helmert transformation'), ('queue_mosaic_cog', 'Can queue creation of mosaic COG'), ('queue_mosaic_xyz', 'Can queue creation of mosaic XYZ tileset'))}, + ), + migrations.AddField( + model_name='layerset', + name='mosaic_geotiff_date', + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name='layerset', + name='multimask_date', + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name='layerset', + name='xyz_tiles_date', + field=models.DateTimeField(blank=True, null=True), + ), + ] diff --git a/ohmg/core/migrations/0015_remove_layerset_mosaic_geotiff_date_and_more.py b/ohmg/core/migrations/0015_remove_layerset_mosaic_geotiff_date_and_more.py new file mode 100644 index 00000000..dced664b --- /dev/null +++ b/ohmg/core/migrations/0015_remove_layerset_mosaic_geotiff_date_and_more.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.27 on 2026-06-23 18:07 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0014_alter_permissions_options_and_more'), + ] + + operations = [ + migrations.RemoveField( + model_name='layerset', + name='mosaic_geotiff_date', + ), + migrations.RemoveField( + model_name='layerset', + name='xyz_tiles_date', + ), + ] diff --git a/ohmg/core/models/layerset.py b/ohmg/core/models/layerset.py index a61ecb5b..38e43022 100644 --- a/ohmg/core/models/layerset.py +++ b/ohmg/core/models/layerset.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Iterable, Union from django.conf import settings +from django.contrib.contenttypes.models import ContentType from django.contrib.gis.db import models from django.contrib.gis.geos import MultiPolygon, Polygon from django.contrib.postgres.fields import ArrayField @@ -70,6 +71,7 @@ class Meta: null=True, ) tilejson = models.JSONField(null=True, blank=True) + multimask_date = models.DateTimeField(blank=True, null=True) def __str__(self): return f"{self.map} - {self.category}" @@ -86,6 +88,28 @@ def layer_display_list(self): def get_layers(self) -> Iterable["Layer"]: return self.layer_set.all() + def get_latest_cog_job(self): + from ohmg.georeference.models import Job + + ct = ContentType.objects.get_for_model(self) + job = ( + Job.objects.filter(operation="layerset_to_cog", target_type=ct, target_id=self.pk) + .order_by("-date_queued") + .first() + ) + return job + + def get_latest_xyz_job(self): + from ohmg.georeference.models import Job + + ct = ContentType.objects.get_for_model(self) + job = ( + Job.objects.filter(operation="layerset_to_xyz", target_type=ct, target_id=self.pk) + .order_by("-date_queued") + .first() + ) + return job + @property def xyz_tiles_url(self): if self.xyz_tiles_prefix: @@ -97,6 +121,17 @@ def xyz_tiles_url(self): else: return None + @property + def xyz_tiles_download_url(self): + if self.xyz_tiles_prefix: + if settings.ENABLE_S3_STORAGE: + base_url = f"{settings.AWS_S3_ENDPOINT_URL}/{settings.AWS_STORAGE_BUCKET_NAME}" + else: + base_url = f"{settings.SITEURL.rstrip('/')}{settings.MEDIA_URL}" + return f"{base_url.rstrip('/')}/{self.xyz_tiles_prefix}/" + else: + return None + @cached_property def centroid(self): return Polygon.from_bbox(self.extent).centroid @@ -146,6 +181,38 @@ def multimask_geojson(self) -> dict: fc["features"].append(mask_geojson) return fc + def queue_mosaic_cog(self) -> int: + """Creates a new job to generate a mosaic cog from this layerset. + If a job already exists for this action, it is re-queued. + + Returns the job id.""" + from ohmg.georeference.models import Job + + j, created = Job.objects.get_or_create( + stage="queued", + operation="layerset_to_cog", + target_id=self.pk, + target_type=ContentType.objects.get_for_model(self), + ) + j.enqueue() + return j.pk + + def queue_mosaic_tileset(self) -> int: + """Creates a new job to generate a XYZ tileset from this layerset. + If a job already exists for this action, it is re-queued. + + Returns the job id.""" + from ohmg.georeference.models import Job + + j, created = Job.objects.get_or_create( + stage="queued", + operation="layerset_to_xyz", + target_id=self.pk, + target_type=ContentType.objects.get_for_model(self), + ) + j.enqueue() + return j.pk + def save(self, set_tilejson: bool = False, *args, **kwargs): if self._state.adding is False: extents = self.get_layers().values_list("extent", flat=True) diff --git a/ohmg/core/models/permissions.py b/ohmg/core/models/permissions.py index 2b68bbb3..912ba342 100644 --- a/ohmg/core/models/permissions.py +++ b/ohmg/core/models/permissions.py @@ -16,4 +16,8 @@ class Meta: default_permissions = () # disable "add", "change", "delete" # and "view" default permissions - permissions = (("use_helmert", "Can use Helmert transformation"),) + permissions = ( + ("use_helmert", "Can use Helmert transformation"), + ("queue_mosaic_cog", "Can queue creation of mosaic COG"), + ("queue_mosaic_xyz", "Can queue creation of mosaic XYZ tileset"), + ) diff --git a/ohmg/core/utils/s3.py b/ohmg/core/utils/s3.py index 743a6b59..515c7ae3 100644 --- a/ohmg/core/utils/s3.py +++ b/ohmg/core/utils/s3.py @@ -1,10 +1,12 @@ +from pathlib import Path + from django.conf import settings + def get_boto3_s3_client(): import boto3 if settings.AWS_ACCESS_KEY_ID: - return boto3.client( "s3", region_name=settings.AWS_S3_REGION_NAME, @@ -14,3 +16,26 @@ def get_boto3_s3_client(): ) else: return None + + +def upload_file_to_bucket(local_path, bucket_path, client=None): + if not client: + client = get_boto3_s3_client() + client.upload_file(local_path, settings.AWS_STORAGE_BUCKET_NAME, bucket_path) + + +# def upload_directory_to_bucket(local_path, bucket_path, client=None): +# if not client: +# client = get_boto3_s3_client() + + +def upload_directory_to_bucket(local_path: Path, bucket_path: str, client=None): + if not client: + client = get_boto3_s3_client() + for file_path in local_path.rglob("*"): + if file_path.is_file(): + # Get relative path, format for S3 (replace backslashes) + s3_key = str(file_path.relative_to(local_path)).replace("\\", "/") + if bucket_path: + s3_key = f"{bucket_path}/{s3_key}" + upload_file_to_bucket(file_path, s3_key, client=client) diff --git a/ohmg/core/views.py b/ohmg/core/views.py index baf30225..fe6fee49 100644 --- a/ohmg/core/views.py +++ b/ohmg/core/views.py @@ -1,5 +1,6 @@ import json import logging +from datetime import datetime from urllib.parse import quote from django.conf import settings @@ -25,6 +26,7 @@ JsonResponseFail, JsonResponseNotFound, JsonResponseSuccess, + JsonResponseUnauthorized, generate_ohmg_context, validate_post_request, ) @@ -415,7 +417,16 @@ def get(self, request, pk, derivative, resource=""): class LayerSetView(View): - @method_decorator(validate_post_request(operations=["bulk-classify-layers", "set-mask"])) + @method_decorator( + validate_post_request( + operations=[ + "bulk-classify-layers", + "set-mask", + "queue-cog-creation", + "queue-tileset-creation", + ] + ) + ) def post(self, request): body = json.loads(request.body) operation = body.get("operation") @@ -475,9 +486,29 @@ def post(self, request): logger.debug(f"removing mask from layer {layer.slug} ({layer.pk})") layer.mask = None layer.save(set_extent=False, skip_map_lookup_update=True) + layerset.multimask_date = datetime.now() + layerset.save() layerset.map.update_item_lookup() return JsonResponseSuccess() + if operation == "queue-cog-creation": + if not request.user.has_perm("core.queue_mosaic_cog"): + return JsonResponseUnauthorized() + layerset = LayerSet.objects.get( + map_id=payload["map-id"], category__slug=payload["category"] + ) + job_id = layerset.queue_mosaic_cog() + return JsonResponseSuccess(payload={"job": job_id}) + + if operation == "queue-tileset-creation": + if not request.user.has_perm("core.queue_mosaic_xyz"): + return JsonResponseUnauthorized() + layerset = LayerSet.objects.get( + map_id=payload["map-id"], category__slug=payload["category"] + ) + job_id = layerset.queue_mosaic_tileset() + return JsonResponseSuccess(payload={"job": job_id}) + class LayersetDerivativeView(View): def get(self, request, mapid, category, derivative): diff --git a/ohmg/frontend/static/css/fancy-icons.css b/ohmg/frontend/static/css/fancy-icons.css index 15807af4..c6a9ee7b 100644 --- a/ohmg/frontend/static/css/fancy-icons.css +++ b/ohmg/frontend/static/css/fancy-icons.css @@ -14,6 +14,7 @@ --pushpin-svg: url(/static/icons/pushpin-60.svg); --envelope-svg: url(/static/icons/envelope-60.svg); --clipboard-svg: url(/static/icons/clipboard-60.svg); + --wrench-svg: url(/static/icons/noun-spanner-937070.svg); } i.fancy { @@ -120,3 +121,8 @@ i.fancy-xs { mask-image: var(--clipboard-svg); -webkit-mask-image: var(--clipboard-svg); } + +.i-wrench { + mask-image: var(--wrench-svg); + -webkit-mask-image: var(--wrench-svg); +} diff --git a/ohmg/frontend/static/fonts/fira-code/fira-code.css b/ohmg/frontend/static/fonts/fira-code/fira-code.css new file mode 100644 index 00000000..12202ee9 --- /dev/null +++ b/ohmg/frontend/static/fonts/fira-code/fira-code.css @@ -0,0 +1,63 @@ +/* cyrillic-ext */ +@font-face { + font-family: 'Fira Code'; + font-style: normal; + font-weight: 300 700; + font-display: swap; + src: url(./uU9NCBsR6Z2vfE9aq3bh0NSDulI.woff2) format('woff2'); + unicode-range: U+0460-052F, U+1C80-1C8A, U+20B4, U+2DE0-2DFF, U+A640-A69F, U+FE2E-FE2F; +} +/* cyrillic */ +@font-face { + font-family: 'Fira Code'; + font-style: normal; + font-weight: 300 700; + font-display: swap; + src: url(./uU9NCBsR6Z2vfE9aq3bh2dSDulI.woff2) format('woff2'); + unicode-range: U+0301, U+0400-045F, U+0490-0491, U+04B0-04B1, U+2116; +} +/* greek-ext */ +@font-face { + font-family: 'Fira Code'; + font-style: normal; + font-weight: 300 700; + font-display: swap; + src: url(./uU9NCBsR6Z2vfE9aq3bh0dSDulI.woff2) format('woff2'); + unicode-range: U+1F00-1FFF; +} +/* greek */ +@font-face { + font-family: 'Fira Code'; + font-style: normal; + font-weight: 300 700; + font-display: swap; + src: url(./uU9NCBsR6Z2vfE9aq3bh3tSDulI.woff2) format('woff2'); + unicode-range: U+0370-0377, U+037A-037F, U+0384-038A, U+038C, U+038E-03A1, U+03A3-03FF; +} +/* symbols2 */ +@font-face { + font-family: 'Fira Code'; + font-style: normal; + font-weight: 300 700; + font-display: swap; + src: url(./uU9NCBsR6Z2vfE9aq3bhZ_Wmh2uX.woff2) format('woff2'); + unicode-range: U+2000-2001, U+2004-2008, U+200A, U+23B8-23BD, U+2500-259F; +} +/* latin-ext */ +@font-face { + font-family: 'Fira Code'; + font-style: normal; + font-weight: 300 700; + font-display: swap; + src: url(./uU9NCBsR6Z2vfE9aq3bh09SDulI.woff2) format('woff2'); + unicode-range: U+0100-02BA, U+02BD-02C5, U+02C7-02CC, U+02CE-02D7, U+02DD-02FF, U+0304, U+0308, U+0329, U+1D00-1DBF, U+1E00-1E9F, U+1EF2-1EFF, U+2020, U+20A0-20AB, U+20AD-20C0, U+2113, U+2C60-2C7F, U+A720-A7FF; +} +/* latin */ +@font-face { + font-family: 'Fira Code'; + font-style: normal; + font-weight: 300 700; + font-display: swap; + src: url(./uU9NCBsR6Z2vfE9aq3bh3dSD.woff2) format('woff2'); + unicode-range: U+0000-00FF, U+0131, U+0152-0153, U+02BB-02BC, U+02C6, U+02DA, U+02DC, U+0304, U+0308, U+0329, U+2000-206F, U+20AC, U+2122, U+2191, U+2193, U+2212, U+2215, U+FEFF, U+FFFD; +} diff --git a/ohmg/frontend/static/fonts/fira-code/uU9NCBsR6Z2vfE9aq3bh09SDulI.woff2 b/ohmg/frontend/static/fonts/fira-code/uU9NCBsR6Z2vfE9aq3bh09SDulI.woff2 new file mode 100644 index 00000000..eac15ae2 Binary files /dev/null and b/ohmg/frontend/static/fonts/fira-code/uU9NCBsR6Z2vfE9aq3bh09SDulI.woff2 differ diff --git a/ohmg/frontend/static/fonts/fira-code/uU9NCBsR6Z2vfE9aq3bh0NSDulI.woff2 b/ohmg/frontend/static/fonts/fira-code/uU9NCBsR6Z2vfE9aq3bh0NSDulI.woff2 new file mode 100644 index 00000000..23c93fe6 Binary files /dev/null and b/ohmg/frontend/static/fonts/fira-code/uU9NCBsR6Z2vfE9aq3bh0NSDulI.woff2 differ diff --git a/ohmg/frontend/static/fonts/fira-code/uU9NCBsR6Z2vfE9aq3bh0dSDulI.woff2 b/ohmg/frontend/static/fonts/fira-code/uU9NCBsR6Z2vfE9aq3bh0dSDulI.woff2 new file mode 100644 index 00000000..522f3e1a Binary files /dev/null and b/ohmg/frontend/static/fonts/fira-code/uU9NCBsR6Z2vfE9aq3bh0dSDulI.woff2 differ diff --git a/ohmg/frontend/static/fonts/fira-code/uU9NCBsR6Z2vfE9aq3bh2dSDulI.woff2 b/ohmg/frontend/static/fonts/fira-code/uU9NCBsR6Z2vfE9aq3bh2dSDulI.woff2 new file mode 100644 index 00000000..de53c244 Binary files /dev/null and b/ohmg/frontend/static/fonts/fira-code/uU9NCBsR6Z2vfE9aq3bh2dSDulI.woff2 differ diff --git a/ohmg/frontend/static/fonts/fira-code/uU9NCBsR6Z2vfE9aq3bh3dSD.woff2 b/ohmg/frontend/static/fonts/fira-code/uU9NCBsR6Z2vfE9aq3bh3dSD.woff2 new file mode 100644 index 00000000..ed70084f Binary files /dev/null and b/ohmg/frontend/static/fonts/fira-code/uU9NCBsR6Z2vfE9aq3bh3dSD.woff2 differ diff --git a/ohmg/frontend/static/fonts/fira-code/uU9NCBsR6Z2vfE9aq3bh3tSDulI.woff2 b/ohmg/frontend/static/fonts/fira-code/uU9NCBsR6Z2vfE9aq3bh3tSDulI.woff2 new file mode 100644 index 00000000..6c75fd94 Binary files /dev/null and b/ohmg/frontend/static/fonts/fira-code/uU9NCBsR6Z2vfE9aq3bh3tSDulI.woff2 differ diff --git a/ohmg/frontend/static/fonts/fira-code/uU9NCBsR6Z2vfE9aq3bhZ_Wmh2uX.woff2 b/ohmg/frontend/static/fonts/fira-code/uU9NCBsR6Z2vfE9aq3bhZ_Wmh2uX.woff2 new file mode 100644 index 00000000..048e4cc4 Binary files /dev/null and b/ohmg/frontend/static/fonts/fira-code/uU9NCBsR6Z2vfE9aq3bhZ_Wmh2uX.woff2 differ diff --git a/ohmg/frontend/static/fonts/Qw3PZQNVED7rKGKxtqIqX5E-AVSJrOCfjY46_DjQbMZhLw.woff2 b/ohmg/frontend/static/fonts/josefin/Qw3PZQNVED7rKGKxtqIqX5E-AVSJrOCfjY46_DjQbMZhLw.woff2 similarity index 100% rename from ohmg/frontend/static/fonts/Qw3PZQNVED7rKGKxtqIqX5E-AVSJrOCfjY46_DjQbMZhLw.woff2 rename to ohmg/frontend/static/fonts/josefin/Qw3PZQNVED7rKGKxtqIqX5E-AVSJrOCfjY46_DjQbMZhLw.woff2 diff --git a/ohmg/frontend/static/fonts/Qw3PZQNVED7rKGKxtqIqX5E-AVSJrOCfjY46_DjQbMhhLzTs.woff2 b/ohmg/frontend/static/fonts/josefin/Qw3PZQNVED7rKGKxtqIqX5E-AVSJrOCfjY46_DjQbMhhLzTs.woff2 similarity index 100% rename from ohmg/frontend/static/fonts/Qw3PZQNVED7rKGKxtqIqX5E-AVSJrOCfjY46_DjQbMhhLzTs.woff2 rename to ohmg/frontend/static/fonts/josefin/Qw3PZQNVED7rKGKxtqIqX5E-AVSJrOCfjY46_DjQbMhhLzTs.woff2 diff --git a/ohmg/frontend/static/fonts/Qw3PZQNVED7rKGKxtqIqX5E-AVSJrOCfjY46_DjQbMlhLzTs.woff2 b/ohmg/frontend/static/fonts/josefin/Qw3PZQNVED7rKGKxtqIqX5E-AVSJrOCfjY46_DjQbMlhLzTs.woff2 similarity index 100% rename from ohmg/frontend/static/fonts/Qw3PZQNVED7rKGKxtqIqX5E-AVSJrOCfjY46_DjQbMlhLzTs.woff2 rename to ohmg/frontend/static/fonts/josefin/Qw3PZQNVED7rKGKxtqIqX5E-AVSJrOCfjY46_DjQbMlhLzTs.woff2 diff --git a/ohmg/frontend/static/fonts/josephine.css b/ohmg/frontend/static/fonts/josefin/josefin.css similarity index 100% rename from ohmg/frontend/static/fonts/josephine.css rename to ohmg/frontend/static/fonts/josefin/josefin.css diff --git a/ohmg/frontend/static/icons/noun-spanner-937070.svg b/ohmg/frontend/static/icons/noun-spanner-937070.svg new file mode 100644 index 00000000..390e0748 --- /dev/null +++ b/ohmg/frontend/static/icons/noun-spanner-937070.svg @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/ohmg/frontend/svelte_components/package.json b/ohmg/frontend/svelte_components/package.json index 417630bd..143e1693 100644 --- a/ohmg/frontend/svelte_components/package.json +++ b/ohmg/frontend/svelte_components/package.json @@ -23,7 +23,8 @@ "rollup-plugin-css-only": "^3.1.0", "rollup-plugin-livereload": "^2.0.0", "rollup-plugin-svelte": "^7.1.5", - "svelte": "^4.0.0" + "svelte": "^4.0.0", + "typescript": "^5.0.0" }, "dependencies": { "@petamoriken/float16": "3.4.7", @@ -37,6 +38,7 @@ "ol-pmtiles": "^1.0.2", "sirv-cli": "^1.0.0", "svelte-markdown": "^0.4.1", + "svelte-preprocess": "^6.0.5", "svelte-select": "^5.8.3", "svelte-tablesort": "^1.1.0" } diff --git a/ohmg/frontend/svelte_components/pnpm-lock.yaml b/ohmg/frontend/svelte_components/pnpm-lock.yaml index 085eed19..32a87ad8 100644 --- a/ohmg/frontend/svelte_components/pnpm-lock.yaml +++ b/ohmg/frontend/svelte_components/pnpm-lock.yaml @@ -41,6 +41,9 @@ importers: svelte-markdown: specifier: ^0.4.1 version: 0.4.1(svelte@4.2.20) + svelte-preprocess: + specifier: ^6.0.5 + version: 6.0.5(@babel/core@7.29.0)(postcss-load-config@3.1.4(postcss@8.5.8))(postcss@8.5.8)(svelte@4.2.20)(typescript@5.9.3) svelte-select: specifier: ^5.8.3 version: 5.8.3 @@ -99,6 +102,9 @@ importers: svelte: specifier: ^4.0.0 version: 4.2.20 + typescript: + specifier: ^5.0.0 + version: 5.9.3 packages: @@ -1160,6 +1166,43 @@ packages: peerDependencies: svelte: ^4.0.0 + svelte-preprocess@6.0.5: + resolution: {integrity: sha512-sgwew5yV/2eMeQobIWgAxCNarKwiTUDIc3siAUbq3sp0G6ONtzk0W+wJihMdqjbYb3iGU3ubpGv0usnnuXT3qg==} + engines: {node: '>= 18.0.0'} + peerDependencies: + '@babel/core': ^7.10.2 + coffeescript: ^2.5.1 + less: ^3.11.3 || ^4.0.0 + postcss: ^7 || ^8 + postcss-load-config: '>=3' + pug: ^3.0.0 + sass: ^1.26.8 + stylus: '>=0.55' + sugarss: ^2.0.0 || ^3.0.0 || ^4.0.0 + svelte: ^4.0.0 || ^5.0.0-next.100 || ^5.0.0 + typescript: ^5.0.0 || ^6.0.0 + peerDependenciesMeta: + '@babel/core': + optional: true + coffeescript: + optional: true + less: + optional: true + postcss: + optional: true + postcss-load-config: + optional: true + pug: + optional: true + sass: + optional: true + stylus: + optional: true + sugarss: + optional: true + typescript: + optional: true + svelte-select@5.8.3: resolution: {integrity: sha512-nQsvflWmTCOZjssdrNptzfD1Ok45hHVMTL5IHay5DINk7dfu5Er+8KsVJnZMJdSircqtR0YlT4YkCFlxOUhVPA==} @@ -1198,6 +1241,11 @@ packages: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + unzipit@1.4.3: resolution: {integrity: sha512-gsq2PdJIWWGhx5kcdWStvNWit9FVdTewm4SEG7gFskWs+XCVaULt9+BwuoBtJiRE8eo3L1IPAOrbByNLtLtIlg==} engines: {node: '>=12'} @@ -2294,6 +2342,15 @@ snapshots: marked: 5.1.2 svelte: 4.2.20 + svelte-preprocess@6.0.5(@babel/core@7.29.0)(postcss-load-config@3.1.4(postcss@8.5.8))(postcss@8.5.8)(svelte@4.2.20)(typescript@5.9.3): + dependencies: + svelte: 4.2.20 + optionalDependencies: + '@babel/core': 7.29.0 + postcss: 8.5.8 + postcss-load-config: 3.1.4(postcss@8.5.8) + typescript: 5.9.3 + svelte-select@5.8.3: dependencies: svelte-floating-ui: 1.5.8 @@ -2344,6 +2401,8 @@ snapshots: dependencies: prelude-ls: 1.2.1 + typescript@5.9.3: {} + unzipit@1.4.3: dependencies: uzip-module: 1.0.3 diff --git a/ohmg/frontend/svelte_components/rollup.config.js b/ohmg/frontend/svelte_components/rollup.config.js index fadb672f..85378337 100644 --- a/ohmg/frontend/svelte_components/rollup.config.js +++ b/ohmg/frontend/svelte_components/rollup.config.js @@ -4,6 +4,7 @@ import resolve from '@rollup/plugin-node-resolve'; import terser from '@rollup/plugin-terser'; import livereload from 'rollup-plugin-livereload'; import css from 'rollup-plugin-css-only'; +import { sveltePreprocess } from 'svelte-preprocess'; const production = !process.env.ROLLUP_WATCH; @@ -19,6 +20,7 @@ function componentExportDetails(componentName) { }, plugins: [ svelte({ + preprocess: sveltePreprocess(), compilerOptions: { // enable run-time checks when not in production dev: !production, @@ -73,6 +75,7 @@ export default (cliArgs) => { 'Map', 'Sessions', 'Maps', + 'Jobs', 'Profiles', 'LatestBlogPosts', 'MapBrowse', diff --git a/ohmg/frontend/svelte_components/src/components/base/Modal.svelte b/ohmg/frontend/svelte_components/src/components/base/Modal.svelte index 96112bbb..d16d7cbb 100644 --- a/ohmg/frontend/svelte_components/src/components/base/Modal.svelte +++ b/ohmg/frontend/svelte_components/src/components/base/Modal.svelte @@ -1,4 +1,4 @@ - - + + + +

Information here

+ + {#if buttonText} + + {/if} + diff --git a/ohmg/frontend/svelte_components/src/components/interfaces/MultiMask.svelte b/ohmg/frontend/svelte_components/src/components/interfaces/MultiMask.svelte index ec255bc0..1cd08645 100644 --- a/ohmg/frontend/svelte_components/src/components/interfaces/MultiMask.svelte +++ b/ohmg/frontend/svelte_components/src/components/interfaces/MultiMask.svelte @@ -36,6 +36,7 @@ import Modal, { openModal } from '../base/Modal.svelte'; import ModalConfirm from '../base/ModalConfirm.svelte'; + import LoadingEllipsis from '../shared/LoadingEllipsis.svelte'; import { usaExtent } from '../../lib/utils'; import { MapViewer } from '../../lib/viewers'; @@ -61,6 +62,8 @@ let fullExtent = usaExtent; + let loading = false; + function updateLayerArr() { layerLookupArr = []; layerLookupMaskedArr = []; diff --git a/ohmg/frontend/svelte_components/src/components/map/Map.svelte b/ohmg/frontend/svelte_components/src/components/map/Map.svelte index 72dcbff5..2f3352da 100644 --- a/ohmg/frontend/svelte_components/src/components/map/Map.svelte +++ b/ohmg/frontend/svelte_components/src/components/map/Map.svelte @@ -26,7 +26,7 @@ import LoadingMask from '../shared/LoadingMask.svelte'; import MultimaskSection from './MultimaskSection.svelte' - import MosaicDownload from './MosaicDownload.svelte'; + import MosaicDerivatives from './MosaicDerivatives.svelte'; import Details from './Details.svelte'; import MapContributors from '../tables/MapContributors.svelte'; @@ -331,7 +331,6 @@ let regionToSetAsNonMap; let regionToSetAsMap; let regionToSkip; - let regionToUnskip; let undoGeorefLayerId; let processing = false; @@ -510,7 +509,7 @@ {#if mosaicSectionActiveTab == "preview"} {#key previewKey} @@ -528,10 +527,9 @@ bind:multimaskKey {userCanEdit} /> {:else if mosaicSectionActiveTab == "download"} - + mapId={MAP.identifier}/> {/if} @@ -751,7 +749,7 @@ bind:modalExtent bind:modalIsGeospatial bind:reinitModalMap - bind:regionToUnskip + bind:regionToSkip /> {/each} diff --git a/ohmg/frontend/svelte_components/src/components/map/MosaicDerivatives.svelte b/ohmg/frontend/svelte_components/src/components/map/MosaicDerivatives.svelte new file mode 100644 index 00000000..fe4e8c1b --- /dev/null +++ b/ohmg/frontend/svelte_components/src/components/map/MosaicDerivatives.svelte @@ -0,0 +1,288 @@ + + + +

Job submitted. You can track its completion on the jobs page.

+ + {submitQueueRequest('queue-cog-creation')}} +> +

Submit mosaic COG generation to queue?

+ + {submitQueueRequest('queue-tileset-creation')}} +> +

Submit XYZ tileset generation to queue?

+ + +
+

Once layers have been trimmed in the MultiMask they can be combined into a single + mosaic. We provide access to this mosaic in the form of file downloads, web service endpoints, and direct integrations + into other platforms. Read more in the docs

+

If the MultiMask has been updated after one of these artifacts was generated, dates will be shown in + red and you can queue that mosaic to be rebuilt.

+
+{#if loading} + +{/if} +{#each layersets as ls} +{#if ls.layers.length >= 2 || ls.id == "main-content"} +

+ + {`${ls.name} (${ls.layers_masked_ct}/${ls.layers.length} layers masked)`} + + {#if ls.multimask_date} + + masks last updated: {ls.masksDateDisplay} + + {/if} +

+
+
+ Cloud Optimized GeoTIFF + + {ls.cogDateDisplay} + {#if ls.showCogQueueBtn} + + {/if} + +
+
Direct download (.tif)
+ +
TileJSON
+ +
XYZ tile endpoint (dynamic)
+ +
WMS endpoint
+ +
+ Static XYZ Tileset + + {ls.xyzDateDisplay} + {#if ls.showXyzQueueBtn} + + {/if} + +
+
Direct download (gzipped tarfile)
+ +
Tiles endpoint
+ +
+ Extensions... + + always current, unless noted + +
+
Open in OpenHistoricalMap editor (uses XYZ tile endpoint)
+ +
IIIF Georef AnnotationPage
+ +
Open in Allmaps Viewer
+ +
+{/if} +{/each} + + \ No newline at end of file diff --git a/ohmg/frontend/svelte_components/src/components/map/MosaicDownload.svelte b/ohmg/frontend/svelte_components/src/components/map/MosaicDownload.svelte deleted file mode 100644 index 399b5b4a..00000000 --- a/ohmg/frontend/svelte_components/src/components/map/MosaicDownload.svelte +++ /dev/null @@ -1,132 +0,0 @@ - - -
-

- Once layers have been trimmed in the MultiMask, a background process can be run - to combine them into a single mosaic file, which serves as a basis for downloads and web services. - If you see n/a below, the mosaic has not yet been created. You can still access - individual layers through the Georeferenced section, or view the mosaic in Allmaps - (powered by IIIF). -

-
-{#each orderedLayersets as ls} -{#if ls.layers.length >= 1} -

{`${ls.name} (${ls.layers.length} layer${ls.layers.length > 1 ? 's' : ''})`}

-
-
TileJSON
-
- {#if ls.mosaic_cog_url} - {CONTEXT.site_url}map/{MAP.identifier}/{ls.id}/tilejson - {:else} - n/a - {/if} -
-
OpenHistoricalMap iD editor
-
{#if ls.mosaic_cog_url} - {CONTEXT.site_url}map/{MAP.identifier}/{ls.id}/ohm - {:else} - n/a - {/if}
-
Cloud-Optimized GeoTIFF
-
- {#if ls.mosaic_cog_url} - {CONTEXT.site_url}map/{MAP.identifier}/{ls.id}/cog - {:else} - n/a - {/if} -
- {#if ls.xyz_tiles_url} -
XYZ tileset
-
- 
{`${ls.xyz_tiles_url}/{z}/{x}/{y}.png`}
-
- {/if} -
IIIF Georef AnnotationPage (JSON)
-
- {CONTEXT.site_url}iiif/mosaic/{MAP.identifier}/{ls.id}/?trim=true -
-
Allmaps
-
- https://viewer.allmaps.org/?url={encodeURIComponent( - `${CONTEXT.site_url}iiif/mosaic/${MAP.identifier}/${ls.id}/?trim=true`, - )} -
-
-{/if} -{/each} - - \ No newline at end of file diff --git a/ohmg/frontend/svelte_components/src/components/map/MultimaskSection.svelte b/ohmg/frontend/svelte_components/src/components/map/MultimaskSection.svelte index dcc69206..c2ea6a32 100644 --- a/ohmg/frontend/svelte_components/src/components/map/MultimaskSection.svelte +++ b/ohmg/frontend/svelte_components/src/components/map/MultimaskSection.svelte @@ -1,8 +1,10 @@ - +

Masks saved successfully.

- - + +

Error! MultiMask not saved. The following layers have errors:

{@html errMsg}

You must fix these errors before you can save your work. Keep in mind it is sometimes easier to remove and recreate a mask than track down and fix specific issues.

- -

+ +

Select which multimask to work on: + {#if loading} + + {:else} + + + \ No newline at end of file diff --git a/ohmg/frontend/svelte_components/src/components/shared/cards/LayerCard.svelte b/ohmg/frontend/svelte_components/src/components/shared/cards/LayerCard.svelte index 84f15a61..c7908bab 100644 --- a/ohmg/frontend/svelte_components/src/components/shared/cards/LayerCard.svelte +++ b/ohmg/frontend/svelte_components/src/components/shared/cards/LayerCard.svelte @@ -8,7 +8,7 @@ import Link from '../../base/Link.svelte'; import BaseCard from '../../base/Card.svelte'; - import { copyToClipboard } from '../../../lib/utils'; + import CopyableText from '../buttons/CopyableText.svelte'; export let CONTEXT; export let LAYERSET_CATEGORIES; @@ -107,13 +107,7 @@ TileJSON
  • - +
  • OpenHistoricalMap iD diff --git a/ohmg/frontend/svelte_components/src/components/shared/cards/SkippedCard.svelte b/ohmg/frontend/svelte_components/src/components/shared/cards/SkippedCard.svelte index bc680750..cfe269f7 100644 --- a/ohmg/frontend/svelte_components/src/components/shared/cards/SkippedCard.svelte +++ b/ohmg/frontend/svelte_components/src/components/shared/cards/SkippedCard.svelte @@ -12,7 +12,7 @@ export let modalExtent; export let modalIsGeospatial; export let reinitModalMap; - export let regionToUnskip; + export let regionToSkip; @@ -47,7 +47,7 @@ class="is-text-link" title="Move to the Prepared section" on:click={() => { - regionToUnskip = region.id + regionToSkip = region.id openModal('modal-confirm-unskip-region') }} > diff --git a/ohmg/frontend/svelte_components/src/components/shared/cards/UnpreparedCard.svelte b/ohmg/frontend/svelte_components/src/components/shared/cards/UnpreparedCard.svelte index 86058281..93fa2b6f 100644 --- a/ohmg/frontend/svelte_components/src/components/shared/cards/UnpreparedCard.svelte +++ b/ohmg/frontend/svelte_components/src/components/shared/cards/UnpreparedCard.svelte @@ -74,7 +74,7 @@
    {#if document.loading_file}
      -
    • loading
      • +
    • loading
      • {#if CONTEXT.user.is_authenticated} + {#if allowRefresh} + + {/if} +
    +
    + {#if paginate} +
    + +
    + {/if} +
    +
    • + {#if showFilters} +
    +
    + + +
    +
    + +
    +
    + {/if} +
    + {#if items.length > 0} +
    + + + + + + + + + + + + + + {#each items as s} + + + + + + + + + + {/each} + +
    IdTargetOperationStageQueuedStartedDuration
    {s.target.name}{s.operation}{s.stage}{timestampToShortString(s.date_queued)}{timestampToShortString(s.date_started)}{secondsToHHMMSS(s.run_duration)}
    +
    + {:else} +
    +
    + {loading ? 'loading...' : 'no results'} +
    +
    + {/if} +
    + + + diff --git a/ohmg/frontend/svelte_components/src/jobs.js b/ohmg/frontend/svelte_components/src/jobs.js new file mode 100644 index 00000000..f9a906e2 --- /dev/null +++ b/ohmg/frontend/svelte_components/src/jobs.js @@ -0,0 +1,6 @@ +import Jobs from './components/tables/Jobs.svelte'; + +export default new Jobs({ + target: document.getElementById('jobs-target'), + props: JSON.parse(document.getElementById('jobs-props').textContent), +}); diff --git a/ohmg/frontend/svelte_components/src/lib/utils.js b/ohmg/frontend/svelte_components/src/lib/utils.js index a91d8b3c..4f6fa145 100644 --- a/ohmg/frontend/svelte_components/src/lib/utils.js +++ b/ohmg/frontend/svelte_components/src/lib/utils.js @@ -122,7 +122,6 @@ export function copyToClipboard(elementId) { const copyText = document.getElementById(elementId); copyText.select(); navigator.clipboard.writeText(copyText.value); - alert('Copied the text: ' + copyText.value); } export function makeSatelliteLayer(apiKey) { diff --git a/ohmg/frontend/svelte_components/src/types/global.d.ts b/ohmg/frontend/svelte_components/src/types/global.d.ts new file mode 100644 index 00000000..e69de29b diff --git a/ohmg/frontend/svelte_components/tsconfig.json b/ohmg/frontend/svelte_components/tsconfig.json new file mode 100644 index 00000000..d304d135 --- /dev/null +++ b/ohmg/frontend/svelte_components/tsconfig.json @@ -0,0 +1,7 @@ +{ + "compilerOptions": { + "verbatimModuleSyntax": true + }, + "include": ["src/**/*"], + "exclude": ["**/*.spec.ts"] +} \ No newline at end of file diff --git a/ohmg/frontend/templates/base.html b/ohmg/frontend/templates/base.html index f895d97b..508be117 100644 --- a/ohmg/frontend/templates/base.html +++ b/ohmg/frontend/templates/base.html @@ -47,7 +47,9 @@ - + + + diff --git a/ohmg/frontend/templates/frontend/jobs.html b/ohmg/frontend/templates/frontend/jobs.html new file mode 100644 index 00000000..d5675372 --- /dev/null +++ b/ohmg/frontend/templates/frontend/jobs.html @@ -0,0 +1,17 @@ +{% extends "base.html" %} +{% load i18n %} +{% load static %} + +{% block title %}Jobs | {{ block.super }}{% endblock %} + +{% block main_content %} + +{% include '_titlebar.html' with title="Jobs" subtitle="see also: sessions" lead_icon="wrench" %} + +
    +
    + {{ JOBS_PROPS|json_script:'jobs-props'}} + {% include '_svelte_component.html' with component_name='Jobs' %} +
    +
    +{% endblock %} diff --git a/ohmg/frontend/urls.py b/ohmg/frontend/urls.py index ff0b7fa7..19baa0b4 100644 --- a/ohmg/frontend/urls.py +++ b/ohmg/frontend/urls.py @@ -11,6 +11,7 @@ ActivityView, Browse, HomePage, + JobsView, PageView, ) @@ -20,6 +21,7 @@ ## app urls paths path("", HomePage.as_view(), name="home"), path("activity/", ActivityView.as_view(), name="activity"), + path("jobs/", JobsView.as_view(), name="jobs"), path("search/", Browse.as_view(), name="search"), path("/", PageView.as_view(), name="page-view"), ## conventional url paths diff --git a/ohmg/frontend/views.py b/ohmg/frontend/views.py index f210c4aa..3d7f8e8a 100644 --- a/ohmg/frontend/views.py +++ b/ohmg/frontend/views.py @@ -90,6 +90,21 @@ def get(self, request): return render(request, "frontend/activity.html", context=context_dict) +class JobsView(View): + def get(self, request): + ohmg_context = generate_ohmg_context(request) + context_dict = { + "CONTEXT": ohmg_context, + "JOBS_PROPS": { + "CONTEXT": ohmg_context, + "showThumbs": True, + "limit": "25", + }, + } + + return render(request, "frontend/jobs.html", context=context_dict) + + class PageView(View): def get(self, request, page): m_date = page.date_modified.strftime("%B %d, %Y") diff --git a/ohmg/georeference/admin.py b/ohmg/georeference/admin.py index ee4e4c97..a5c48cd8 100644 --- a/ohmg/georeference/admin.py +++ b/ohmg/georeference/admin.py @@ -4,6 +4,7 @@ GCP, GCPGroup, GeorefSession, + Job, PrepSession, SessionLock, ) @@ -55,3 +56,36 @@ class SessionLockAdmin(admin.ModelAdmin): admin.site.register(SessionLock, SessionLockAdmin) + + +@admin.action(description="Enqueue selected jobs") +def enqueue_job(modeladmin, request, queryset): + for obj in queryset.all(): + obj.enqueue() + + +class JobAdmin(admin.ModelAdmin): + readonly_fields = ( + "stage", + "date_created", + "date_queued", + "date_started", + "date_ended", + "run_duration", + ) + list_display = ( + "target", + "operation", + "stage", + "date_created", + "date_queued", + "date_started", + "date_ended", + "message", + "run_duration", + ) + list_filter = ("operation", "stage", "map") + actions = [enqueue_job] + + +admin.site.register(Job, JobAdmin) diff --git a/ohmg/georeference/management/commands/mosaic.py b/ohmg/georeference/management/commands/mosaic.py index 979a86e5..3aa51287 100644 --- a/ohmg/georeference/management/commands/mosaic.py +++ b/ohmg/georeference/management/commands/mosaic.py @@ -59,7 +59,7 @@ def handle(self, *args, **options): if options.background: create_mosaic_tileset.delay(ls.pk) else: - m.generate_xyz_tiles(ls, use_multiprocessing=options.multiprocessing) + m.generate_tileset(ls, use_multiprocessing=options.multiprocessing) m.cleanup_files() if options.operation == "generate-cog": diff --git a/ohmg/georeference/management/commands/sessions.py b/ohmg/georeference/management/commands/session.py similarity index 90% rename from ohmg/georeference/management/commands/sessions.py rename to ohmg/georeference/management/commands/session.py index 1f16808e..4c44460a 100644 --- a/ohmg/georeference/management/commands/sessions.py +++ b/ohmg/georeference/management/commands/session.py @@ -1,10 +1,8 @@ +import json + from django.core.management.base import BaseCommand -from ohmg.georeference.models import ( - GeorefSession, - PrepSession, - SessionBase -) +from ohmg.georeference.models import GeorefSession, PrepSession, SessionBase from ohmg.georeference.sessions import delete_expired_session_locks @@ -19,6 +17,7 @@ def add_arguments(self, parser): "undo", "list", "delete-expired", + "data", ], help="specify the operation to carry out", ) @@ -54,7 +53,7 @@ def _model_from_type(self, session_type): def handle(self, *args, **options): operation = options["operation"] - if operation in ["run", "undo"]: + if operation in ["run", "undo", "data"]: bs = SessionBase.objects.get(pk=options["pk"]) model = self._model_from_type(bs.type) session = model.objects.get(pk=options["pk"]) @@ -63,6 +62,8 @@ def handle(self, *args, **options): session.run() elif operation == "undo": session.undo() + elif operation == "data": + print(json.dumps(session.data, indent=1)) elif operation == "list": if options["type"]: diff --git a/ohmg/georeference/migrations/0003_job.py b/ohmg/georeference/migrations/0003_job.py new file mode 100644 index 00000000..01aee1c7 --- /dev/null +++ b/ohmg/georeference/migrations/0003_job.py @@ -0,0 +1,35 @@ +# Generated by Django 4.2.27 on 2026-06-05 15:28 + +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0012_layerset_xyz_tiles_prefix'), + ('contenttypes', '0002_remove_content_type_name'), + ('georeference', '0002_sessionbase_map'), + ] + + operations = [ + migrations.CreateModel( + name='Job', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('date_created', models.DateTimeField(default=django.utils.timezone.now)), + ('date_queued', models.DateTimeField(blank=True, null=True)), + ('date_started', models.DateTimeField(blank=True, null=True)), + ('date_ended', models.DateTimeField(blank=True, null=True)), + ('operation', models.CharField(choices=[('load_documents', 'load_documents'), ('split_document', 'split_document'), ('region_to_cog', 'region_to_cog'), ('layerset_to_cog', 'layerset_to_cog'), ('layerset_to_xyz', 'layerset_to_xyz')], max_length=25)), + ('stage', models.CharField(blank=True, choices=[('queued', 'queued'), ('running', 'running'), ('completed', 'completed'), ('errored', 'errored')], max_length=25, null=True)), + ('run_duration', models.IntegerField(blank=True, null=True)), + ('target_id', models.PositiveIntegerField()), + ('data', models.JSONField(blank=True, default=dict)), + ('message', models.CharField(blank=True, max_length=255, null=True)), + ('map', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='core.map')), + ('target_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.contenttype')), + ], + ), + ] diff --git a/ohmg/georeference/models.py b/ohmg/georeference/models.py index 6b84b34b..196d6c62 100644 --- a/ohmg/georeference/models.py +++ b/ohmg/georeference/models.py @@ -20,16 +20,17 @@ from ohmg.core.models import ( Document, Layer, + LayerSet, Map, Region, RegionCategory, ) from ohmg.core.utils import ( - full_reverse, random_alnum, ) from ohmg.georeference.georeferencer import Georeferencer from ohmg.georeference.splitter import Splitter +from ohmg.georeference.tasks import create_mosaic_cog, create_mosaic_tileset from .sessions import ( add_lock, @@ -213,11 +214,6 @@ def save_from_geojson(self, geojson, region, transformation=None): return group -def set_upload_location(instance, filename): - """this function has to return the location to upload the file""" - return os.path.join(f"{instance.type}s", filename) - - def get_default_session_data(session_type): """Return a dict of the keys/types for a sessions's data field. Also used for type-checking during validation.""" @@ -382,43 +378,6 @@ def extend_locks(self): for lock in self.locks.all(): lock.extend() - def serialize(self): - # handle the non- js-serializable attributes - doc_id, layer_alt, d_create, d_mod, d_run = None, None, None, None, None - d_run_d, d_run_t = None, None - if self.doc: - doc_id = self.doc.pk - if self.lyr: - layer_alt = self.lyr.pk - if self.date_created: - d_create = self.date_created.strftime("%Y-%m-%d - %H:%M") - if self.date_modified: - d_mod = self.date_modified.strftime("%Y-%m-%d - %H:%M") - if self.date_run: - d_run = self.date_run.strftime("%Y-%m-%d - %H:%M") - d_run_d = self.date_run.strftime("%Y-%m-%d") - d_run_t = self.date_run.strftime("%H:%M") - - return { - "id": self.pk, - "type": self.get_type_display(), - "document": doc_id, - "layer": layer_alt, - "stage": self.stage, - "status": self.status, - "note": self.note, - "data": self.data, - "user": { - "name": self.user.username, - "profile": full_reverse("profile_detail", args=(self.user.username,)), - }, - "date_created": d_create, - "date_modified": d_mod, - "date_run": d_run, - "date_run_date": d_run_d, - "date_run_time": d_run_t, - } - def update_stage(self, stage, save=True): self.stage = stage logger.info(f"{self.__str__()} | stage: {self.stage}") @@ -801,3 +760,131 @@ def __str__(self): def extend(self): self.expiration += timedelta(seconds=settings.GEOREFERENCE_SESSION_LENGTH) self.save() + + +JOB_OPERATIONS = ( + ("load_documents", "load_documents"), + ("split_document", "split_document"), + ("region_to_cog", "region_to_cog"), + ("layerset_to_cog", "layerset_to_cog"), + ("layerset_to_xyz", "layerset_to_xyz"), +) + +JOB_STAGES = ( + ("queued", "queued"), + ("running", "running"), + ("completed", "completed"), + ("errored", "errored"), +) + + +class Job(models.Model): + """Submits and tracks backend operations.""" + + date_created = models.DateTimeField( + default=timezone.now, + ) + date_queued = models.DateTimeField(blank=True, null=True) + date_started = models.DateTimeField(blank=True, null=True) + date_ended = models.DateTimeField(blank=True, null=True) + operation = models.CharField( + max_length=25, + choices=JOB_OPERATIONS, + ) + stage = models.CharField( + max_length=25, + choices=JOB_STAGES, + blank=True, + null=True, + ) + run_duration = models.IntegerField( + blank=True, + null=True, + ) + target_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) + target_id = models.PositiveIntegerField() + target = GenericForeignKey("target_type", "target_id") + data = models.JSONField( + default=dict, + blank=True, + ) + map = models.ForeignKey( + Map, + models.SET_NULL, + null=True, + blank=True, + ) + message = models.CharField( + blank=True, + null=True, + max_length=255, + ) + + def validate(self): + """Test the requested operation against the attached resource + and the input data. Return dict with valid bool and message.""" + + valid = True + message = None + if self.operation in ["layerset_to_cog", "layerset_to_xyz"]: + if not isinstance(self.target, LayerSet): + valid = False + message = f"invalid target {self.target} for {self.operation} job" + else: + valid = False + message = "invalid operation (or operation not yet supported)" + + return {"success": valid, "message": message} + + def run(self): + """Run the operation, there is bespoke logic here for each operation.""" + + validation = self.validate() + if not validation["success"]: + self.stage = "errored" + self.message = validation["message"] + self.save() + return + + self.start() + if self.operation == "layerset_to_cog": + create_mosaic_cog.delay(self.target.pk, self.pk) + elif self.operation == "layerset_to_xyz": + create_mosaic_tileset.delay(self.target.pk, self.pk) + + def enqueue(self): + self.stage = "queued" + self.date_queued = timezone.now() + self.date_started = None + self.date_ended = None + self.run_duration = None + self.save() + + def start(self): + self.stage = "running" + self.date_started = timezone.now() + self.save() + + def end(self, success: bool = True, message: str = ""): + self.date_ended = timezone.now() + self.message = message + self.stage = "completed" if success else "errored" + elapsed_delta = self.date_ended - self.date_started + self.run_duration = elapsed_delta.seconds + self.save() + + def save(self, *args, **kwargs): + if not self.map: + if isinstance( + self.target, + ( + Document, + Region, + Layer, + LayerSet, + ), + ): + self.map = self.target.map + if isinstance(self.target, Map): + self.maps = self.target + return super(Job, self).save(*args, **kwargs) diff --git a/ohmg/georeference/mosaicker.py b/ohmg/georeference/mosaicker.py index 323ab5eb..c8d8f901 100644 --- a/ohmg/georeference/mosaicker.py +++ b/ohmg/georeference/mosaicker.py @@ -61,7 +61,7 @@ def generate_mosaic_vrt(self, layerset) -> VRTHandler: for feature in multimask_geojson["features"]: layer_name = feature["properties"]["layer"] - print(layer_name) + logger.debug(layer_name) try: layer = Layer.objects.get(slug=layer_name, region__document__map=layerset.map) except Layer.MultipleObjectsReturned: @@ -102,18 +102,16 @@ def generate_mosaic_vrt(self, layerset) -> VRTHandler: outputBounds=bounds, separate=False, ) - print("building vrt") + logger.info("building mosaic vrt") self.mosaic_vrt = VRTHandler(f"{layerset.map.identifier}-{layerset.category.slug}") trim_list = [str(i.get_path()) for i in self.trimmed_vrts] gdal.BuildVRT(str(self.mosaic_vrt.get_path()), trim_list, options=vo) def generate_cog(self, layerset: LayerSet): - start = datetime.now() - self.generate_mosaic_vrt(layerset) - print("building final geotiff") + logger.info("begin writing mosaic geotiff") to = gdal.TranslateOptions( format="COG", @@ -139,9 +137,7 @@ def generate_cog(self, layerset: LayerSet): layerset.save(set_tilejson=True) - print(f"completed - elapsed time: {datetime.now() - start}") - - def generate_xyz_tiles( + def generate_tileset( self, layerset: LayerSet, min_zoom: int = 13, @@ -154,7 +150,9 @@ def generate_xyz_tiles( self.generate_mosaic_vrt(layerset) in_path = self.mosaic_vrt.get_path() - prefix = f"tiles/{layerset.map.identifier}/{layerset.category.slug}/{random_alnum()}" + prefix = f"tiles/{layerset.map.identifier}/{layerset.category.slug}/" + # prefix += f"{datetime.now().strftime('%Y%m%d')}__{random_alnum()}" + prefix += datetime.now().strftime("%Y%m%d%H%M%S") logger.info(f"creating new tileset {prefix}") logger.info(f"source dataset: {in_path}") diff --git a/ohmg/georeference/tasks.py b/ohmg/georeference/tasks.py index 8d0f80c1..3403b495 100644 --- a/ohmg/georeference/tasks.py +++ b/ohmg/georeference/tasks.py @@ -9,17 +9,13 @@ from ohmg.core.models import LayerSet from ohmg.core.utils.s3 import get_boto3_s3_client -from .models import ( - GeorefSession, - PrepSession, -) -from .sessions import delete_expired_session_locks - logger = logging.getLogger(__name__) @app.task def run_preparation_session(sessionid): + from .models import PrepSession + session = PrepSession.objects.get(pk=sessionid) session.run() return session.pk @@ -27,6 +23,8 @@ def run_preparation_session(sessionid): @app.task def bulk_run_preparation_sessions(sessionids): + from .models import PrepSession + for sessionid in sessionids: session = PrepSession.objects.get(pk=sessionid) session.run() @@ -35,6 +33,8 @@ def bulk_run_preparation_sessions(sessionids): @app.task def run_georeference_session(sessionid): + from .models import GeorefSession + session = GeorefSession.objects.get(pk=sessionid) session.run() return session.pk @@ -42,6 +42,8 @@ def run_georeference_session(sessionid): @app.task def delete_stale_sessions(): + from .sessions import delete_expired_session_locks + delete_expired_session_locks() @@ -52,29 +54,48 @@ def delete_preview_vrts(id): @app.task -def create_mosaic_cog(layersetid): - from ohmg.georeference.mosaicker import Mosaicker +def create_mosaic_cog(layersetid: int, jobid: int | None = None): + from .models import Job + from .mosaicker import Mosaicker + logger.debug("begin create mosaic task") try: layerset = LayerSet.objects.get(pk=layersetid) except LayerSet.DoesNotExist: logger.warning(f"LayerSet does not exist: {layersetid}. Cancelling mosaic creation.") - m = Mosaicker() - m.generate_cog(layerset) - m.cleanup_files() + try: + m = Mosaicker() + success = True + message = m.generate_cog(layerset) + m.cleanup_files() + except Exception as e: + success = False + message = e + + if jobid: + Job.objects.get(pk=jobid).end(success=success, message=message) @app.task -def create_mosaic_tileset(layersetid): - from ohmg.georeference.mosaicker import Mosaicker +def create_mosaic_tileset(layersetid: int, jobid: int | None = None): + from .models import Job + from .mosaicker import Mosaicker try: layerset = LayerSet.objects.get(pk=layersetid) except LayerSet.DoesNotExist: logger.warning(f"LayerSet does not exist: {layersetid}. Cancelling mosaic creation.") - m = Mosaicker() - m.generate_xyz_tiles(layerset) - m.cleanup_files() + try: + m = Mosaicker() + success = True + message = m.generate_tileset(layerset) + m.cleanup_files() + except Exception as e: + success = False + message = e + + if jobid: + Job.objects.get(pk=jobid).end(success=success, message=message) @app.task @@ -94,3 +115,24 @@ def cleanup_existing_tileset(prefix): ) except FileNotFoundError: pass + + +@app.task +def run_queued_mosaic_jobs(): + from ohmg.georeference.models import Job + + max_jobs_ct = settings.MAX_CONCURRENT_MOSAIC_JOBS + + mosaic_jobs = Job.objects.filter(operation__in=["layerset_to_cog", "layerset_to_xyz"]) + + ## if there are already max mosaic jobs running, don't start another one + running_jobs_ct = mosaic_jobs.filter(stage="running").count() + spots_left = max_jobs_ct - running_jobs_ct + started = [] + if spots_left: + queued_jobs = mosaic_jobs.filter(stage="queued").order_by("date_queued") + for job in queued_jobs[:spots_left]: + logger.info(f"starting queued job {job.pk}: {job.operation}, {job.target}") + job.run() + started.append(job.pk) + return {"jobs": started} diff --git a/ohmg/georeference/urls.py b/ohmg/georeference/urls.py index f4bc8858..730409cc 100644 --- a/ohmg/georeference/urls.py +++ b/ohmg/georeference/urls.py @@ -1,10 +1,6 @@ from django.urls import path -from .views import ( - GeoreferenceView, - SessionView, - SplitView, -) +from .views import GeoreferenceView, JobView, SessionView, SplitView urlpatterns = [ path("split/", SplitView.as_view(), name="base_split_view"), @@ -15,4 +11,5 @@ name="georeference_view", ), path("session//", SessionView.as_view(), name="session_view"), + path("job//", JobView.as_view(), name="job_view"), ] diff --git a/ohmg/georeference/utils.py b/ohmg/georeference/utils.py index 14fe14e9..864cc9e3 100644 --- a/ohmg/georeference/utils.py +++ b/ohmg/georeference/utils.py @@ -1,6 +1,8 @@ import io import logging import os +import shutil +import tarfile from datetime import datetime from multiprocessing import Pool from pathlib import Path @@ -10,7 +12,11 @@ from django.conf import settings from rio_tiler.io import Reader -from ohmg.core.utils.s3 import get_boto3_s3_client +from ohmg.core.utils.s3 import ( + get_boto3_s3_client, + upload_directory_to_bucket, + upload_file_to_bucket, +) logger = logging.getLogger(__name__) @@ -56,6 +62,7 @@ def make_xyz_tiles_with_multiprocessing( min_zoom: int = 13, max_zoom: int = 20, ): + raise NotImplementedError("Tileset generation with multiprocessing is not fully implemented") start = datetime.now() logger.info(f"creating new tileset with multiprocessing {prefix}") @@ -88,8 +95,7 @@ def make_xyz_tiles( max_zoom: int = 20, ): start = datetime.now() - - logger.info(f"creating new tileset {prefix}") + logger.info(f"creating new tileset {prefix} from {data_source}") progress_pct = { 10: False, @@ -103,9 +109,7 @@ def make_xyz_tiles( 90: False, } - if settings.ENABLE_S3_STORAGE: - s3 = get_boto3_s3_client() - + tmp_tileset_root = Path(settings.TEMP_DIR, prefix) with Reader(data_source) as src: zooms = range(min_zoom, max_zoom + 1) bounds = src.geographic_bounds @@ -118,21 +122,11 @@ def make_xyz_tiles( ## only make a tile if there is valid data (skip empty tiles) if tile.data_as_image().any(): rendered_bytes = tile.render() - if settings.ENABLE_S3_STORAGE: - key = f"{prefix}/{coords.z}/{coords.x}/{coords.y}.png" - file_like = io.BytesIO(rendered_bytes) - s3.upload_fileobj( - file_like, - settings.AWS_STORAGE_BUCKET_NAME, - key, - ) - else: - out_root = Path(settings.MEDIA_ROOT, prefix) - out_dir = Path(out_root, str(coords.z), str(coords.x)) - out_dir.mkdir(parents=True, exist_ok=True) - file_path = Path(out_dir, f"{coords.y}.png") - with open(file_path, "wb") as file: - file.write(rendered_bytes) + out_dir = Path(tmp_tileset_root, str(coords.z), str(coords.x)) + out_dir.mkdir(parents=True, exist_ok=True) + file_path = Path(out_dir, f"{coords.y}.png") + with open(file_path, "wb") as file: + file.write(rendered_bytes) ## progress logging tiles_written_ct += 1 pct = int((tiles_written_ct / tiles_total_ct) * 100) @@ -141,6 +135,33 @@ def make_xyz_tiles( logger.debug(f"{prefix} {k}% written") progress_pct[k] = True - logger.info(f"{prefix} completed, elapsed time: {datetime.now() - start}") + logger.info(f"tileset {prefix} created, elapsed time: {datetime.now() - start}") + + start2 = datetime.now() + logger.info(f"creating gzip archive for tileset {prefix}") + + tmp_gz_path = Path(tmp_tileset_root.parent, "archive.tar.gz") + print(tmp_gz_path) + print(tmp_tileset_root.name) + with tarfile.open(tmp_gz_path, "w:gz") as tar: + tar.add(tmp_tileset_root, arcname=tmp_tileset_root.name) + logger.info(f"gzip {tmp_gz_path.name} created, elapsed time: {datetime.now() - start2}") + + logger.debug("copying tileset to final location") + + if settings.ENABLE_S3_STORAGE: + s3 = get_boto3_s3_client() + upload_directory_to_bucket(tmp_tileset_root, prefix, client=s3) + # place the archive file within the top-level of the tileset itself, + # alongside the z-level folders + upload_file_to_bucket(tmp_gz_path, f"{prefix}/{tmp_gz_path.name}", client=s3) + else: + local_media_dest = Path(settings.MEDIA_ROOT, prefix) + # local_media_dest.parent.mkdir(exist_ok=True, parents=True) + shutil.copytree(tmp_tileset_root, local_media_dest, dirs_exist_ok=True) + shutil.copyfile(tmp_gz_path, Path(local_media_dest, tmp_gz_path.name)) + + os.remove(tmp_gz_path) + shutil.rmtree(tmp_tileset_root) return prefix diff --git a/ohmg/georeference/views.py b/ohmg/georeference/views.py index 49f88c2e..42989883 100644 --- a/ohmg/georeference/views.py +++ b/ohmg/georeference/views.py @@ -17,6 +17,7 @@ JsonResponseFail, JsonResponseNotFound, JsonResponseSuccess, + JsonResponseUnauthorized, generate_ohmg_context, validate_post_request, ) @@ -29,11 +30,7 @@ from ohmg.core.utils.performance import time_this_function from .georeferencer import Georeferencer -from .models import ( - GeorefSession, - PrepSession, - SessionBase, -) +from .models import GeorefSession, Job, PrepSession, SessionBase from .splitter import Splitter from .tasks import ( bulk_run_preparation_sessions, @@ -361,3 +358,17 @@ def post(self, request, sessionid): except Exception as e: return JsonResponseFail(e) return JsonResponseSuccess() + + +class JobView(View): + @method_decorator(validate_post_request(operations=["queue"])) + def post(self, request, jobid): + if not request.user.is_superuser: + return JsonResponseUnauthorized() + job = get_object_or_404(Job, pk=jobid) + body = json.loads(request.body) + operation = body.get("operation") + + if operation == "queue": + job.enqueue() + return JsonResponseSuccess() diff --git a/zensical.toml b/zensical.toml index 1371616e..74523066 100644 --- a/zensical.toml +++ b/zensical.toml @@ -60,6 +60,10 @@ nav = [ "guides/preparation.md", "guides/georeferencing.md", "guides/trimming.md", + "guides/generating-mosaics.md", + ]}, + { "Using georeferenced maps" = [ + "guides/services.md", ]}, { "Walkthroughs" = [ "walkthroughs/new-iberia-la-1885.md"