Skip to content

Commit 768c9a1

Browse files
jonavellecuerdojonavellecuerdo
committed
1 parent b01146f commit 768c9a1

14 files changed

Lines changed: 135 additions & 110 deletions

File tree

‎.pre-commit-config.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,6 @@ repos:
2424
types: ["python"]
2525
- id: pip-audit
2626
name: pip-audit
27-
entry: uv run pip-audit --ignore-vuln PYSEC-2024-271
27+
entry: uv run pip-audit
2828
language: system
2929
pass_filenames: false

‎Makefile‎

Lines changed: 37 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -22,16 +22,20 @@ help: # Preview Makefile commands
2222
# Python environment and dependency commands
2323
##############################################
2424

25-
install: .venv .git/hooks/pre-commit # Install Python dependencies and create virtual environment if not exists
25+
install: .venv .git/hooks/pre-commit # Install Python dependencies and create virtual environment if needed
2626
uv sync --dev
2727

2828
.venv: # Creates virtual environment if not found
2929
@echo "Creating virtual environment at .venv..."
3030
uv venv .venv
3131

32-
.git/hooks/pre-commit: # Sets up pre-commit hook if not setup
33-
@echo "Installing pre-commit hooks..."
34-
uv run pre-commit install
32+
.git/hooks/pre-commit: # Sets up pre-commit commit hooks if not setup
33+
@echo "Installing pre-commit commit hooks..."
34+
uv run pre-commit install --hook-type pre-commit
35+
36+
.git/hooks/pre-push: # Sets up pre-commit push hooks if not setup
37+
@echo "Installing pre-commit push hooks..."
38+
uv run pre-commit install --hook-type pre-push
3539

3640
venv: .venv # Create the Python virtual environment
3741

@@ -52,42 +56,25 @@ coveralls: test # Write coverage data to an LCOV report
5256
uv run coverage lcov -o ./coverage/lcov.info
5357

5458
####################################
55-
# Code quality and safety commands
59+
# Code linting and formatting
5660
####################################
5761

58-
lint: black mypy ruff safety # Run linters
59-
60-
black: # Run 'black' linter and print a preview of suggested changes
61-
uv run black --check --diff .
62-
63-
mypy: # Run 'mypy' linter
62+
lint: # Run linting, alerts only, no code changes
63+
uv run ruff format --diff
6464
uv run mypy .
65-
66-
ruff: # Run 'ruff' linter and print a preview of errors
6765
uv run ruff check .
6866

69-
safety: # Check for security vulnerabilities and verify Pipfile.lock is up-to-date
70-
uv run pip-audit
71-
72-
lint-apply: black-apply ruff-apply # Apply changes with 'black' and resolve 'fixable errors' with 'ruff'
73-
74-
black-apply: # Apply changes with 'black'
75-
uv run black .
76-
77-
ruff-apply: # Resolve 'fixable errors' with 'ruff'
67+
lint-fix: # Run linting, auto fix behaviors where supported
68+
uv run ruff format .
7869
uv run ruff check --fix .
7970

80-
####################################
81-
# MinIO local S3 commands
82-
####################################
83-
84-
start-minio-server:
85-
docker compose --env-file .env -f $(MINIO_COMPOSE_FILE) up -d
71+
security: # Run security / vulnerability checks
72+
uv run pip-audit
8673

87-
stop-minio-server:
88-
docker compose --env-file .env -f $(MINIO_COMPOSE_FILE) stop
8974

90-
### Terraform-generated Developer Deploy Commands for Dev environment ###
75+
###############################################
76+
# Docker image, ECR, and Lambda Management
77+
###############################################
9178
check-arch:
9279
@ARCH_FILE=".aws-architecture"; \
9380
if [[ "$(CPU_ARCH)" != "linux/amd64" && "$(CPU_ARCH)" != "linux/arm64" ]]; then \
@@ -99,19 +86,18 @@ check-arch:
9986
echo "latest" > .arch_tag; \
10087
fi
10188

102-
dist-dev: check-arch ## Build docker container (intended for developer-based manual build)
89+
dist-dev: check-arch # Build docker container (intended for developer-based manual build)
10390
@ARCH_TAG=$$(cat .arch_tag); \
10491
docker buildx inspect $(ECR_NAME_DEV) >/dev/null 2>&1 || docker buildx create --name $(ECR_NAME_DEV) --use; \
10592
docker buildx use $(ECR_NAME_DEV); \
10693
docker buildx build --platform $(CPU_ARCH) \
107-
--load \
108-
--tag $(ECR_URL_DEV):$$ARCH_TAG \
109-
--tag $(ECR_URL_DEV):make-$$ARCH_TAG \
110-
--tag $(ECR_URL_DEV):make-$(shell git describe --always) \
111-
--tag $(ECR_NAME_DEV):$$ARCH_TAG \
112-
.
113-
114-
publish-dev: dist-dev ## Build, tag and push (intended for developer-based manual publish)
94+
--load \
95+
--tag $(ECR_URL_DEV):$$ARCH_TAG \
96+
--tag $(ECR_URL_DEV):make-$$ARCH_TAG \
97+
--tag $(ECR_URL_DEV):make-$(shell git describe --always) \
98+
--tag $(ECR_NAME_DEV):$$ARCH_TAG \
99+
100+
publish-dev: dist-dev # Build, tag and push (intended for developer-based manual publish)
115101
@ARCH_TAG=$$(cat .arch_tag); \
116102
aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin $(ECR_URL_DEV); \
117103
docker push $(ECR_URL_DEV):$$ARCH_TAG; \
@@ -120,15 +106,7 @@ publish-dev: dist-dev ## Build, tag and push (intended for developer-based manua
120106
echo "Cleaning up dangling Docker images..."; \
121107
docker image prune -f --filter "dangling=true"
122108

123-
### If this is a Lambda repo, uncomment the two lines below ###
124-
# update-lambda-dev: ## Updates the lambda with whatever is the most recent image in the ecr (intended for developer-based manual update)
125-
# @ARCH_TAG=$$(cat .arch_tag); \
126-
# aws lambda update-function-code \
127-
# --region us-east-1 \
128-
# --function-name $(FUNCTION_DEV) \
129-
# --image-uri $(ECR_URL_DEV):make-$$ARCH_TAG
130-
131-
docker-clean: ## Clean up Docker detritus
109+
docker-clean: # Clean up Docker detritus
132110
@ARCH_TAG=$$(cat .arch_tag); \
133111
echo "Cleaning up Docker leftovers (containers, images, builders)"; \
134112
docker rmi -f $(ECR_URL_DEV):$$ARCH_TAG; \
@@ -137,3 +115,13 @@ docker-clean: ## Clean up Docker detritus
137115
docker rmi -f $(ECR_NAME_DEV):$$ARCH_TAG || true; \
138116
docker buildx rm $(ECR_NAME_DEV) || true
139117
@rm -rf .arch_tag
118+
119+
####################################
120+
# MinIO local S3 commands
121+
####################################
122+
123+
start-minio-server:
124+
docker compose --env-file .env -f $(MINIO_COMPOSE_FILE) up -d
125+
126+
stop-minio-server:
127+
docker compose --env-file .env -f $(MINIO_COMPOSE_FILE) stop

‎dsc/db/models.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -143,8 +143,11 @@ def create(self) -> "ItemSubmissionDB":
143143
)
144144
logger.info(
145145
"Created record "
146-
f"{ITEM_SUBMISSION_LOG_STR.format(batch_id=self.batch_id,
147-
item_identifier=self.item_identifier)}"
146+
f"{
147+
ITEM_SUBMISSION_LOG_STR.format(
148+
batch_id=self.batch_id, item_identifier=self.item_identifier
149+
)
150+
}"
148151
)
149152
except PutError as exception:
150153
# if the `PutError` is due to failing conditional check, this means

‎dsc/item_submission.py‎

Lines changed: 66 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -157,10 +157,14 @@ def _from_db(cls, item_submission_db: ItemSubmissionDB) -> ItemSubmission:
157157
if hasattr(item_submission_db, attr):
158158
value = getattr(item_submission_db, attr)
159159
setattr(item_submission, attr, value)
160-
logger.debug(f"Populated record {ITEM_SUBMISSION_LOG_STR.format(
161-
batch_id=item_submission_db.batch_id,
162-
item_identifier=item_submission_db.item_identifier
163-
)}")
160+
logger.debug(
161+
f"Populated record {
162+
ITEM_SUBMISSION_LOG_STR.format(
163+
batch_id=item_submission_db.batch_id,
164+
item_identifier=item_submission_db.item_identifier,
165+
)
166+
}"
167+
)
164168
return item_submission
165169

166170
def save(self) -> None:
@@ -176,9 +180,14 @@ def save(self) -> None:
176180
)
177181
item_submission_db.create()
178182

179-
logger.info(f"Saved record " f"{ITEM_SUBMISSION_LOG_STR.format(
180-
batch_id=self.batch_id, item_identifier=self.item_identifier
181-
)}")
183+
logger.info(
184+
f"Saved record "
185+
f"{
186+
ITEM_SUBMISSION_LOG_STR.format(
187+
batch_id=self.batch_id, item_identifier=self.item_identifier
188+
)
189+
}"
190+
)
182191

183192
def upsert_db(self) -> None:
184193
"""Upsert a record in DynamoDB from ItemSubmission.
@@ -195,9 +204,14 @@ def upsert_db(self) -> None:
195204
)
196205
item_submission_db.save()
197206

198-
logger.info(f"Upserted record " f"{ITEM_SUBMISSION_LOG_STR.format(
199-
batch_id=self.batch_id, item_identifier=self.item_identifier
200-
)}")
207+
logger.info(
208+
f"Upserted record "
209+
f"{
210+
ITEM_SUBMISSION_LOG_STR.format(
211+
batch_id=self.batch_id, item_identifier=self.item_identifier
212+
)
213+
}"
214+
)
201215

202216
def ready_to_submit(self) -> bool:
203217
"""Check if the item submission is ready to be submitted."""
@@ -208,39 +222,62 @@ def ready_to_submit(self) -> bool:
208222
match self.status:
209223
case ItemSubmissionStatus.INGEST_SUCCESS:
210224
logger.info(
211-
f"Record {ITEM_SUBMISSION_LOG_STR.format(batch_id=self.batch_id,
212-
item_identifier=self.item_identifier)
213-
} " "already ingested, skipping submission"
225+
f"Record {
226+
ITEM_SUBMISSION_LOG_STR.format(
227+
batch_id=self.batch_id, item_identifier=self.item_identifier
228+
)
229+
} "
230+
"already ingested, skipping submission"
214231
)
215232
case ItemSubmissionStatus.SUBMIT_SUCCESS:
216233
logger.info(
217-
f"Record " f"{ITEM_SUBMISSION_LOG_STR.format(batch_id=self.batch_id,
218-
item_identifier=self.item_identifier)
219-
} " " already submitted, skipping submission"
234+
f"Record "
235+
f"{
236+
ITEM_SUBMISSION_LOG_STR.format(
237+
batch_id=self.batch_id, item_identifier=self.item_identifier
238+
)
239+
} "
240+
" already submitted, skipping submission"
220241
)
221242
case ItemSubmissionStatus.CREATE_FAILED | ItemSubmissionStatus.CREATE_SKIPPED:
222243
logger.info(
223-
f"Record " f"{ITEM_SUBMISSION_LOG_STR.format(batch_id=self.batch_id,
224-
item_identifier=self.item_identifier)
225-
} " " submission assets invalid, skipping submission"
244+
f"Record "
245+
f"{
246+
ITEM_SUBMISSION_LOG_STR.format(
247+
batch_id=self.batch_id, item_identifier=self.item_identifier
248+
)
249+
} "
250+
" submission assets invalid, skipping submission"
226251
)
227252
case ItemSubmissionStatus.MAX_RETRIES_REACHED:
228253
logger.info(
229-
f"Record " f"{ITEM_SUBMISSION_LOG_STR.format(batch_id=self.batch_id,
230-
item_identifier=self.item_identifier)
231-
} " "max retries reached, skipping submission"
254+
f"Record "
255+
f"{
256+
ITEM_SUBMISSION_LOG_STR.format(
257+
batch_id=self.batch_id, item_identifier=self.item_identifier
258+
)
259+
} "
260+
"max retries reached, skipping submission"
232261
)
233262
case None:
234263
logger.info(
235-
f"Record " f"{ITEM_SUBMISSION_LOG_STR.format(batch_id=self.batch_id,
236-
item_identifier=self.item_identifier)
237-
} " " status unknown, skipping submission"
264+
f"Record "
265+
f"{
266+
ITEM_SUBMISSION_LOG_STR.format(
267+
batch_id=self.batch_id, item_identifier=self.item_identifier
268+
)
269+
} "
270+
" status unknown, skipping submission"
238271
)
239272
case _:
240273
logger.debug(
241-
f"Record " f"{ITEM_SUBMISSION_LOG_STR.format(batch_id=self.batch_id,
242-
item_identifier=self.item_identifier)
243-
} " "allowed for submission"
274+
f"Record "
275+
f"{
276+
ITEM_SUBMISSION_LOG_STR.format(
277+
batch_id=self.batch_id, item_identifier=self.item_identifier
278+
)
279+
} "
280+
"allowed for submission"
244281
)
245282
ready_to_submit = True
246283

@@ -301,12 +338,11 @@ def create_dspace_metadata(
301338

302339
for field_name, field_mapping in metadata_mapping.items():
303340
if field_name != "item_identifier":
304-
305341
field_value = item_metadata.get(field_mapping["source_field_name"])
306342
if not field_value and field_mapping.get("required", False):
307343
raise ItemMetadataMissingRequiredFieldError(
308344
"Item metadata missing required field: '"
309-
f"{field_mapping["source_field_name"]}'"
345+
f"{field_mapping['source_field_name']}'"
310346
)
311347

312348
if field_value:

‎dsc/reports/base.py‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ def prepare_attachments(self) -> list[tuple]:
107107

108108
def upload_attachments(self, output_location: str) -> None:
109109
for filename, buffer in self.prepare_attachments():
110-
file = f"{output_location.removesuffix("/")}/{filename}"
110+
file = f"{output_location.removesuffix('/')}/{filename}"
111111
mode = "wb" if isinstance(buffer, BytesIO) else "w"
112112
with smart_open.open(file, mode) as f:
113113
f.write(buffer.getvalue())
@@ -149,7 +149,6 @@ def create_item_submissions_csv(self, fields: list[str] | None = None) -> String
149149

150150

151151
class CreateReport(Report):
152-
153152
@property
154153
def subject(self) -> str:
155154
return f"DSC Create Batch Results - {self.workflow_name}, batch='{self.batch_id}'"
@@ -168,7 +167,6 @@ def generate_summary(self) -> str:
168167

169168

170169
class SubmitReport(Report):
171-
172170
@property
173171
def subject(self) -> str:
174172
return f"DSC Submit Results - {self.workflow_name}, batch='{self.batch_id}'"
@@ -192,7 +190,6 @@ def generate_summary(self) -> str:
192190

193191

194192
class FinalizeReport(Report):
195-
196193
@property
197194
def subject(self) -> str:
198195
return f"[{CONFIG.workspace}] DSpace Ingest Results - {self.workflow_name}, batch='{self.batch_id}'" # noqa: E501

‎dsc/reports/digitized_theses.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88

99

1010
class DigitizedThesesFinalizeReport(FinalizeReport):
11-
1211
attachments = (
1312
*FinalizeReport.attachments,
1413
Attachment(

‎dsc/utils/aws/s3.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ def files_iter(
125125
):
126126
continue
127127

128-
yield f"s3://{bucket}/{content["Key"]}"
128+
yield f"s3://{bucket}/{content['Key']}"
129129

130130

131131
def run_aws_cli_sync(

‎dsc/utils/aws/sqs.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ def send(
135135
MessageAttributes=message_attributes,
136136
MessageBody=message_body,
137137
)
138-
logger.debug(f"Sent message: {response["MessageId"]}")
138+
logger.debug(f"Sent message: {response['MessageId']}")
139139
return response
140140

141141
def receive(self) -> Iterator[MessageTypeDef]:
@@ -151,7 +151,7 @@ def receive(self) -> Iterator[MessageTypeDef]:
151151
)
152152
if "Messages" in response:
153153
for message in response["Messages"]:
154-
logger.debug(f"Retrieved message: {message["MessageId"]}")
154+
logger.debug(f"Retrieved message: {message['MessageId']}")
155155
message_count += 1
156156
yield message
157157
else:

0 commit comments

Comments
 (0)