diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..ad39586 --- /dev/null +++ b/.env.example @@ -0,0 +1,6 @@ +AWS_ACCESS_KEY_ID=test +AWS_SECRET_ACCESS_KEY=test +AWS_DEFAULT_REGION=us-east-1 +AWS_ENDPOINT_URL=http://localstack:4566 + +QUEUE_NAME=development_events_reindex_daily diff --git a/build.sh b/build.sh index fd797b8..af0451c 100755 --- a/build.sh +++ b/build.sh @@ -29,3 +29,4 @@ packagePythonLambda 'check_links' packagePythonLambda 'analytics_worker' packagePythonLambda 'analytics_queue_reports' packagePythonLambda 'datafile_generator' +packagePythonLambda 'events_queue_reindex_touched_dois' diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..5ae377f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,19 @@ +services: + lambda: + image: python:3.14.6-alpine + environment: + - AWS_ACCESS_KEY_ID=test + - AWS_SECRET_ACCESS_KEY=test + - AWS_DEFAULT_REGION=us-east-1 + - AWS_ENDPOINT_URL=http://localstack:4566 + - QUEUE_NAME + networks: + - localstack_network + volumes: + - ${LAMBDA_DIR}:/var/task + working_dir: /var/task + command: sh -c "pip install --quiet -r requirements.txt && python ${LAMBDA_PATH}" + +networks: + localstack_network: + external: true diff --git a/lambdas/events_queue_reindex_touched_dois/events_queue_reindex_touched_dois_runner.py b/lambdas/events_queue_reindex_touched_dois/events_queue_reindex_touched_dois_runner.py new file mode 100644 index 0000000..70ba9d4 --- /dev/null +++ b/lambdas/events_queue_reindex_touched_dois/events_queue_reindex_touched_dois_runner.py @@ -0,0 +1,43 @@ +""" +This lambda function is triggered by a CloudWatch event rule on a schedule. +It add a message to the queue with the current date, to be processed by Events. +Events will call the `reindex_touched_dois` method to reindex the DOIs touched today. +""" + +import os +import json +import datetime +import boto3 + +# Lambda handler +def lambda_handler(event, context): + """Lambda process handler""" + print("Starting reindex_touched_dois_runner lambda function") + + # Configuration + queue_name = os.getenv('QUEUE_NAME') + + # Calculate the date range for the reindex + yesterday = datetime.datetime.now(datetime.UTC) - datetime.timedelta(days=1) + yesterday = yesterday.strftime("%Y-%m-%d") + + message = { + 'Name': 'shoryuken_class', + 'Type': 'String', + 'Value': 'ReindexTouchedDoisWorker', + + 'date': yesterday, + } + + # Queue a task for each repository + sqs = boto3.resource('sqs') + queue = sqs.get_queue_by_name(QueueName=queue_name) + queue.send_message(MessageBody=json.dumps(message)) + + return("Queued reindex for dois touched on {}".format(yesterday)) + +if __name__ == '__main__': + # For local testing fake the arguments to lambda handler function + event = {} + context = [] + lambda_handler(event, context) diff --git a/lambdas/events_queue_reindex_touched_dois/requirements.txt b/lambdas/events_queue_reindex_touched_dois/requirements.txt new file mode 100644 index 0000000..30ddf82 --- /dev/null +++ b/lambdas/events_queue_reindex_touched_dois/requirements.txt @@ -0,0 +1 @@ +boto3 diff --git a/run_lambda.sh b/run_lambda.sh new file mode 100755 index 0000000..bfac52a --- /dev/null +++ b/run_lambda.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: + ./run_lambda.sh + +Additional environment variables need to be added to docker-compose.yml +EOF +} + +[[ $# -eq 1 ]] || { usage; exit 1; } + +SCRIPT_PATH="$1" +[[ -f "$SCRIPT_PATH" ]] || { echo "Error: file not found: $SCRIPT_PATH" >&2; exit 1; } + +LAMBDA_DIR="$(dirname "$(realpath "$SCRIPT_PATH")")" +LAMBDA_PATH="$(basename "$SCRIPT_PATH")" +export LAMBDA_DIR LAMBDA_PATH + +docker compose run --rm lambda