diff --git a/distributions/demo-server/build.gradle b/distributions/demo-server/build.gradle index abca66dcd..9f3143944 100644 --- a/distributions/demo-server/build.gradle +++ b/distributions/demo-server/build.gradle @@ -82,6 +82,7 @@ dependencies { compile project(':extensions:auth:portability-auth-twitter') compile project(':extensions:auth:portability-auth-imgur') compile project(':extensions:auth:portability-auth-koofr') + compile project(':extensions:auth:portability-auth-synology') compile project(':extensions:data-transfer:portability-data-transfer-apple') compile project(':extensions:data-transfer:portability-data-transfer-daybook') @@ -94,6 +95,7 @@ dependencies { compile project(':extensions:data-transfer:portability-data-transfer-rememberthemilk') compile project(':extensions:data-transfer:portability-data-transfer-smugmug') compile project(':extensions:data-transfer:portability-data-transfer-spotify') + compile project(':extensions:data-transfer:portability-data-transfer-synology') compile project(':extensions:data-transfer:portability-data-transfer-twitter') compile project(':extensions:data-transfer:portability-data-transfer-imgur') compile project(':extensions:data-transfer:portability-data-transfer-koofr') diff --git a/extensions/auth/portability-auth-google/src/main/java/org/datatransferproject/auth/google/GoogleOAuthConfig.java b/extensions/auth/portability-auth-google/src/main/java/org/datatransferproject/auth/google/GoogleOAuthConfig.java index ff04a129e..c28a23fb6 100644 --- a/extensions/auth/portability-auth-google/src/main/java/org/datatransferproject/auth/google/GoogleOAuthConfig.java +++ b/extensions/auth/portability-auth-google/src/main/java/org/datatransferproject/auth/google/GoogleOAuthConfig.java @@ -20,6 +20,7 @@ import static org.datatransferproject.types.common.models.DataVertical.CALENDAR; import static org.datatransferproject.types.common.models.DataVertical.CONTACTS; import static org.datatransferproject.types.common.models.DataVertical.MAIL; +import static org.datatransferproject.types.common.models.DataVertical.MEDIA; import static org.datatransferproject.types.common.models.DataVertical.MUSIC; import static org.datatransferproject.types.common.models.DataVertical.PHOTOS; import static org.datatransferproject.types.common.models.DataVertical.SOCIAL_POSTS; @@ -70,6 +71,7 @@ public Map> getExportScopes() { .put(SOCIAL_POSTS, ImmutableSet.of("https://www.googleapis.com/auth/plus.login")) .put(TASKS, ImmutableSet.of("https://www.googleapis.com/auth/tasks.readonly")) .put(VIDEOS, ImmutableSet.of("https://www.googleapis.com/auth/photoslibrary.readonly")) + .put(MEDIA, ImmutableSet.of("https://www.googleapis.com/auth/photoslibrary.readonly")) .put(MUSIC, ImmutableSet.of("https://www.googleapis.com/auth/music")) .build(); } @@ -85,6 +87,7 @@ public Map> getImportScopes() { .put(PHOTOS, ImmutableSet.of("https://www.googleapis.com/auth/photoslibrary.appendonly")) .put(TASKS, ImmutableSet.of("https://www.googleapis.com/auth/tasks")) .put(VIDEOS, ImmutableSet.of("https://www.googleapis.com/auth/photoslibrary")) + .put(MEDIA, ImmutableSet.of("https://www.googleapis.com/auth/photoslibrary")) .put(MUSIC, ImmutableSet.of("https://www.googleapis.com/auth/music")) .build(); } diff --git a/extensions/auth/portability-auth-synology/build.gradle b/extensions/auth/portability-auth-synology/build.gradle new file mode 100644 index 000000000..147f5c099 --- /dev/null +++ b/extensions/auth/portability-auth-synology/build.gradle @@ -0,0 +1,13 @@ +plugins { + id 'maven' + id 'signing' +} + +dependencies { + compile project(':portability-spi-api') + compile project(':portability-spi-cloud') + compile project(':libraries:auth') +} + +configurePublication(project) + diff --git a/extensions/auth/portability-auth-synology/src/main/java/org/datatransferproject/auth/synology/SynologyAuthServiceExtension.java b/extensions/auth/portability-auth-synology/src/main/java/org/datatransferproject/auth/synology/SynologyAuthServiceExtension.java new file mode 100644 index 000000000..ccb1e2899 --- /dev/null +++ b/extensions/auth/portability-auth-synology/src/main/java/org/datatransferproject/auth/synology/SynologyAuthServiceExtension.java @@ -0,0 +1,9 @@ +package org.datatransferproject.auth.synology; + +import org.datatransferproject.auth.OAuth2ServiceExtension; + +public class SynologyAuthServiceExtension extends OAuth2ServiceExtension { + public SynologyAuthServiceExtension() { + super(new SynologyOAuthConfig()); + } +} diff --git a/extensions/auth/portability-auth-synology/src/main/java/org/datatransferproject/auth/synology/SynologyOAuthConfig.java b/extensions/auth/portability-auth-synology/src/main/java/org/datatransferproject/auth/synology/SynologyOAuthConfig.java new file mode 100644 index 000000000..c7e7bfcf8 --- /dev/null +++ b/extensions/auth/portability-auth-synology/src/main/java/org/datatransferproject/auth/synology/SynologyOAuthConfig.java @@ -0,0 +1,51 @@ +package org.datatransferproject.auth.synology; + +import static org.datatransferproject.types.common.models.DataVertical.MEDIA; +import static org.datatransferproject.types.common.models.DataVertical.PHOTOS; +import static org.datatransferproject.types.common.models.DataVertical.VIDEOS; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import java.util.Map; +import java.util.Set; +import org.datatransferproject.auth.OAuth2Config; +import org.datatransferproject.types.common.models.DataVertical; + +public class SynologyOAuthConfig implements OAuth2Config { + private static String AUTH_HOST = "https://identity.synology.com"; + private static String USER_INFO_SCOPE = "userinfo"; + private static String OFFLINE_ACCESS_SCOPE = "offline_access"; + + @Override + public String getServiceName() { + return "Synology"; + } + + @Override + public String getAuthUrl() { + return AUTH_HOST + "/oauth2/auth"; + } + + @Override + public String getTokenUrl() { + return AUTH_HOST + "/oauth2/token"; + } + + @Override + public Map> getExportScopes() { + return ImmutableMap.>builder() + .put(PHOTOS, ImmutableSet.of(USER_INFO_SCOPE, OFFLINE_ACCESS_SCOPE)) + .put(VIDEOS, ImmutableSet.of(USER_INFO_SCOPE, OFFLINE_ACCESS_SCOPE)) + .put(MEDIA, ImmutableSet.of(USER_INFO_SCOPE, OFFLINE_ACCESS_SCOPE)) + .build(); + } + + @Override + public Map> getImportScopes() { + return ImmutableMap.>builder() + .put(PHOTOS, ImmutableSet.of(USER_INFO_SCOPE, OFFLINE_ACCESS_SCOPE)) + .put(VIDEOS, ImmutableSet.of(USER_INFO_SCOPE, OFFLINE_ACCESS_SCOPE)) + .put(MEDIA, ImmutableSet.of(USER_INFO_SCOPE, OFFLINE_ACCESS_SCOPE)) + .build(); + } +} diff --git a/extensions/auth/portability-auth-synology/src/main/java/org/datatransferproject/auth/synology/package-info.java b/extensions/auth/portability-auth-synology/src/main/java/org/datatransferproject/auth/synology/package-info.java new file mode 100644 index 000000000..7c3731bc5 --- /dev/null +++ b/extensions/auth/portability-auth-synology/src/main/java/org/datatransferproject/auth/synology/package-info.java @@ -0,0 +1 @@ +package org.datatransferproject.auth.synology; diff --git a/extensions/auth/portability-auth-synology/src/main/resources/META-INF/services/org.datatransferproject.spi.api.auth.extension.AuthServiceExtension b/extensions/auth/portability-auth-synology/src/main/resources/META-INF/services/org.datatransferproject.spi.api.auth.extension.AuthServiceExtension new file mode 100644 index 000000000..a787e083b --- /dev/null +++ b/extensions/auth/portability-auth-synology/src/main/resources/META-INF/services/org.datatransferproject.spi.api.auth.extension.AuthServiceExtension @@ -0,0 +1 @@ +org.datatransferproject.auth.synology.SynologyAuthServiceExtension diff --git a/extensions/data-transfer/portability-data-transfer-synology/Readme.md b/extensions/data-transfer/portability-data-transfer-synology/Readme.md new file mode 100644 index 000000000..a0ac4983b --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/Readme.md @@ -0,0 +1,23 @@ +# Synology +This folder contains the extension implementation for the +[Synology](https://www.synology.com). + +## Data Supported + +Synology currently supports the following data types: + - Photos + - Videos + +## Current State + + - Photos/Videos: + - Exporting is supported via Synology Photos or Bee Photos. + - Importing is supported via the exporting source platforms (currently only Google Photos). + +## Keys & Auth +Synology uses OAuth 2 for authorization. + +### Create an OAuth App +Contact [Synology DTP Developer Team](dtp@synology.com) for further information. + + diff --git a/extensions/data-transfer/portability-data-transfer-synology/build.gradle b/extensions/data-transfer/portability-data-transfer-synology/build.gradle new file mode 100644 index 000000000..bc1808c6f --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/build.gradle @@ -0,0 +1,30 @@ +/* + * Copyright 2025 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +plugins { + id 'maven' + id 'signing' +} + +dependencies { + compile project(':portability-spi-transfer') + compile project(':portability-spi-cloud') + compile project(':portability-transfer') + compile "com.squareup.okhttp3:okhttp:${okHttpVersion}" +} + +configurePublication(project) diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/SynologyTransferExtension.java b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/SynologyTransferExtension.java new file mode 100644 index 000000000..94990986e --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/SynologyTransferExtension.java @@ -0,0 +1,120 @@ +package org.datatransferproject.datatransfer.synology; + +import static org.datatransferproject.types.common.models.DataVertical.MEDIA; +import static org.datatransferproject.types.common.models.DataVertical.PHOTOS; +import static org.datatransferproject.types.common.models.DataVertical.VIDEOS; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import okhttp3.OkHttpClient; +import org.datatransferproject.api.launcher.ExtensionContext; +import org.datatransferproject.api.launcher.Monitor; +import org.datatransferproject.datatransfer.synology.media.SynologyMediaImporter; +import org.datatransferproject.datatransfer.synology.photos.SynologyPhotosImporter; +import org.datatransferproject.datatransfer.synology.service.SynologyDTPService; +import org.datatransferproject.datatransfer.synology.service.SynologyOAuthTokenManager; +import org.datatransferproject.datatransfer.synology.uploader.SynologyUploader; +import org.datatransferproject.datatransfer.synology.videos.SynologyVideosImporter; +import org.datatransferproject.spi.cloud.storage.AppCredentialStore; +import org.datatransferproject.spi.cloud.storage.JobStore; +import org.datatransferproject.spi.transfer.extension.TransferExtension; +import org.datatransferproject.spi.transfer.idempotentexecutor.IdempotentImportExecutor; +import org.datatransferproject.spi.transfer.idempotentexecutor.IdempotentImportExecutorExtension; +import org.datatransferproject.spi.transfer.provider.Exporter; +import org.datatransferproject.spi.transfer.provider.Importer; +import org.datatransferproject.transfer.JobMetadata; +import org.datatransferproject.types.common.models.DataVertical; +import org.datatransferproject.types.transfer.auth.AppCredentials; +import org.datatransferproject.types.transfer.serviceconfig.TransferServiceConfig; + +public class SynologyTransferExtension implements TransferExtension { + private static final ImmutableList SUPPORTED_SERVICES = + ImmutableList.of(PHOTOS, VIDEOS, MEDIA); + private Monitor monitor; + + private boolean initialized = false; + + private ImmutableMap importerMap; + + @Override + public String getServiceId() { + return "Synology"; + } + + private String getExtensionKey() { + return getServiceId().toUpperCase() + "_KEY"; + } + + private String getExtensionSecret() { + return getServiceId().toUpperCase() + "_SECRET"; + } + + @Override + public Exporter getExporter(DataVertical transferDataType) { + return null; + } + + @Override + public Importer getImporter(DataVertical transferDataType) { + Preconditions.checkArgument(initialized); + Preconditions.checkArgument(SUPPORTED_SERVICES.contains(transferDataType)); + monitor.info(() -> "Getting importer for " + transferDataType); + monitor.info(() -> "has importer: " + importerMap.containsKey(transferDataType)); + return importerMap.get(transferDataType); + } + + @Override + public void initialize(ExtensionContext context) { + if (initialized) { + return; + } + + monitor = context.getMonitor(); + + AppCredentials appCredentials; + try { + appCredentials = + context + .getService(AppCredentialStore.class) + .getAppCredentials(getExtensionKey(), getExtensionSecret()); + + } catch (Exception e) { + monitor.info( + () -> + "Unable to retrieve Synology AppCredentials. Please configure Synology KEY and" + + " SECRET."); + return; + } + + JobStore jobStore = context.getService(JobStore.class); + IdempotentImportExecutor idempotentImportExecutor = + context + .getService(IdempotentImportExecutorExtension.class) + .getRetryingIdempotentImportExecutor(context); + SynologyOAuthTokenManager tokenManager = new SynologyOAuthTokenManager(appCredentials, monitor); + OkHttpClient client = context.getService(OkHttpClient.class); + TransferServiceConfig transferServiceConfig = context.getService(TransferServiceConfig.class); + SynologyDTPService synologyDTPService = + new SynologyDTPService( + monitor, + transferServiceConfig, + JobMetadata.getExportService(), + jobStore, + tokenManager, + client); + SynologyUploader synologyUploader = + new SynologyUploader(idempotentImportExecutor, monitor, synologyDTPService); + + ImmutableMap.Builder importerBuilder = ImmutableMap.builder(); + importerBuilder.put(MEDIA, new SynologyMediaImporter(monitor, tokenManager, synologyUploader)); + importerBuilder.put( + PHOTOS, new SynologyPhotosImporter(monitor, tokenManager, synologyUploader)); + importerBuilder.put( + VIDEOS, new SynologyVideosImporter(monitor, tokenManager, synologyUploader)); + importerMap = importerBuilder.build(); + + monitor.info(() -> "Initializing SynologyTransferExtension"); + initialized = true; + } +} diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/constant/SynologyConstant.java b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/constant/SynologyConstant.java new file mode 100644 index 000000000..e057b3942 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/constant/SynologyConstant.java @@ -0,0 +1,23 @@ +/* + * Copyright 2025 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.datatransferproject.datatransfer.synology.constant; + +/** Defines constants for Synology data transfer. */ +public interface SynologyConstant { + public static final String ALBUM_ITEM_ID_FORMAT = "%s-%s"; +} diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/constant/SynologyErrorCode.java b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/constant/SynologyErrorCode.java new file mode 100644 index 000000000..b8b69aeb4 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/constant/SynologyErrorCode.java @@ -0,0 +1,45 @@ +/* + * Copyright 2025 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.datatransferproject.datatransfer.synology.constant; + +/** + * Defines error codes for Synology data transfer. + * + *

The codes are structured as follows: + * + *

    + *
  • S: Service identifier + *
  • MM: Module identifier + *
  • SS: Specific error code + *
+ * + *

Example: 11001 represents a DTP Client error with the code MAX_RETRIES_EXCEEDED where 1 is the + * service identifier, 10 is the module identifier, and 01 is the specific error code. + */ +public interface SynologyErrorCode { + // [S][MM][SS] + // S: Service identifier + // MM: Module identifier + // SS: Specific error code + + // Common Errors + + // DTP Client Errors + public static final int MAX_RETRIES_EXCEEDED = 11001; + public static final int IMPORT_FAILED = 11002; +} diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/exceptions/SynologyDTPClientException.java b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/exceptions/SynologyDTPClientException.java new file mode 100644 index 000000000..434b89dfd --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/exceptions/SynologyDTPClientException.java @@ -0,0 +1,32 @@ +/* + * Copyright 2025 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.datatransferproject.datatransfer.synology.exceptions; + +/** + * Base class for exceptions thrown by the Synology DTP client. Business logic exceptions should + * extend this class. + */ +public abstract class SynologyDTPClientException extends SynologyException { + protected SynologyDTPClientException(String message, String errorCode) { + super(message, errorCode); + } + + protected SynologyDTPClientException(String message, Throwable cause, String errorCode) { + super(message, cause, errorCode); + } +} diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/exceptions/SynologyException.java b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/exceptions/SynologyException.java new file mode 100644 index 000000000..15251c555 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/exceptions/SynologyException.java @@ -0,0 +1,54 @@ +/* + * Copyright 2025 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.datatransferproject.datatransfer.synology.exceptions; + +/** Exception thrown when an error occurs during a Synology import / export operation. */ +public class SynologyException extends RuntimeException { + private final String errorCode; + + /** + * @param message error message + */ + protected SynologyException(String message) { + super(message); + this.errorCode = null; + } + + /** + * @param message error message + * @param errorCode the error code + */ + protected SynologyException(String message, String errorCode) { + super(message); + this.errorCode = errorCode; + } + + /** + * @param message error message + * @param cause the cause of the exception + * @param errorCode the error code + */ + protected SynologyException(String message, Throwable cause, String errorCode) { + super(message, cause); + this.errorCode = errorCode; + } + + public String getErrorCode() { + return errorCode; + } +} diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/exceptions/SynologyHttpException.java b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/exceptions/SynologyHttpException.java new file mode 100644 index 000000000..4eb598943 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/exceptions/SynologyHttpException.java @@ -0,0 +1,51 @@ +/* + * Copyright 2025 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.datatransferproject.datatransfer.synology.exceptions; + +/** + * Exception thrown by the Synology service. Message contains HTTP status code and response body. + */ +public class SynologyHttpException extends SynologyException { + private final int statusCode; + private final String responseBody; + + /** + * @param message the message + * @param statusCode the status code + * @param responseBody the response body + */ + public SynologyHttpException(String message, int statusCode, String responseBody) { + super(message, "HTTP_" + statusCode); + this.statusCode = statusCode; + this.responseBody = responseBody; + } + + @Override + public String getMessage() { + return String.format( + "%s (statusCode=%d, responseBody=%s)", super.getMessage(), statusCode, responseBody); + } + + public int getStatusCode() { + return statusCode; + } + + public String getResponseBody() { + return responseBody; + } +} diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/exceptions/SynologyImportException.java b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/exceptions/SynologyImportException.java new file mode 100644 index 000000000..574771faf --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/exceptions/SynologyImportException.java @@ -0,0 +1,31 @@ +/* + * Copyright 2025 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.datatransferproject.datatransfer.synology.exceptions; + +import org.datatransferproject.datatransfer.synology.constant.SynologyErrorCode; + +/** Exception thrown when an error occurs during a Synology import process. */ +public class SynologyImportException extends SynologyDTPClientException { + public SynologyImportException(String message) { + super(message, "IMPORT_ERROR"); + } + + public SynologyImportException(String message, Throwable cause) { + super(message, cause, String.valueOf(SynologyErrorCode.IMPORT_FAILED)); + } +} diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/exceptions/SynologyMaxRetriesExceededException.java b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/exceptions/SynologyMaxRetriesExceededException.java new file mode 100644 index 000000000..1eea281d6 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/exceptions/SynologyMaxRetriesExceededException.java @@ -0,0 +1,54 @@ +/* + * Copyright 2025 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.datatransferproject.datatransfer.synology.exceptions; + +import org.datatransferproject.datatransfer.synology.constant.SynologyErrorCode; + +/** + * Exception thrown when the maximum number of retries is exceeded. + * + *

Message includes the number of attempts and the cause of the exception. + */ +public class SynologyMaxRetriesExceededException extends SynologyDTPClientException { + private final int attempts; + + public SynologyMaxRetriesExceededException(String message, int attempts, Throwable cause) { + super(message, cause, String.valueOf(SynologyErrorCode.MAX_RETRIES_EXCEEDED)); + this.attempts = attempts; + } + + @Override + public String getMessage() { + StringBuilder base = new StringBuilder(super.getMessage()); + base.append(" after ").append(attempts).append(" attempts"); + + if (getCause() instanceof SynologyHttpException) { + SynologyHttpException serviceEx = (SynologyHttpException) getCause(); + base.append( + String.format( + " (statusCode=%d, responseBody=%s)", + serviceEx.getStatusCode(), serviceEx.getResponseBody())); + } + + return base.toString(); + } + + public int getAttempts() { + return attempts; + } +} diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/media/SynologyMediaImporter.java b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/media/SynologyMediaImporter.java new file mode 100644 index 000000000..408cd6539 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/media/SynologyMediaImporter.java @@ -0,0 +1,49 @@ +package org.datatransferproject.datatransfer.synology.media; + +import java.util.UUID; +import org.datatransferproject.api.launcher.Monitor; +import org.datatransferproject.datatransfer.synology.service.SynologyOAuthTokenManager; +import org.datatransferproject.datatransfer.synology.uploader.SynologyUploader; +import org.datatransferproject.spi.transfer.idempotentexecutor.IdempotentImportExecutor; +import org.datatransferproject.spi.transfer.provider.ImportResult; +import org.datatransferproject.spi.transfer.provider.Importer; +import org.datatransferproject.types.common.models.media.MediaContainerResource; +import org.datatransferproject.types.transfer.auth.TokensAndUrlAuthData; + +public class SynologyMediaImporter + implements Importer { + private final Monitor monitor; + private final SynologyOAuthTokenManager tokenManager; + private final SynologyUploader synologyUploader; + + public SynologyMediaImporter( + Monitor monitor, SynologyOAuthTokenManager tokenManager, SynologyUploader synologyUploader) { + this.monitor = monitor; + this.tokenManager = tokenManager; + this.synologyUploader = synologyUploader; + + monitor.info(() -> "Creating SynologyMediaImporter"); + } + + @Override + public ImportResult importItem( + UUID jobId, + IdempotentImportExecutor idempotentExecutor, + TokensAndUrlAuthData authData, + MediaContainerResource data) + throws Exception { + monitor.info(() -> "==== [SynologyImporter]: SynologyMediaImporter starts importing item ===="); + tokenManager.addAuthDataIfNotExist(jobId, authData); + + try { + synologyUploader.importAlbums(data.getAlbums(), jobId); + synologyUploader.importPhotos(data.getPhotos(), jobId); + synologyUploader.importVideos(data.getVideos(), jobId); + } catch (Exception e) { + monitor.severe(() -> "[SynologyImporter] SynologyMediaImporter failed to import data:" + e); + return new ImportResult(e); + } + + return ImportResult.OK; + } +} diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/models/C2Api.java b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/models/C2Api.java new file mode 100644 index 000000000..1814abaf5 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/models/C2Api.java @@ -0,0 +1,86 @@ +/* + * Copyright 2025 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.datatransferproject.datatransfer.synology.models; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.datatransferproject.datatransfer.synology.utils.UrlUtils; + +/** Data model for Synology C2 API configuration. */ +public class C2Api extends ServiceConfig.Service { + private final ApiPath apiPath; + + private final String createAlbum; + private final String uploadItem; + private final String addItemToAlbum; + + @JsonCreator + public C2Api(@JsonProperty("baseUrl") String baseUrl, @JsonProperty("apiPath") ApiPath apiPath) { + super(baseUrl); + + this.apiPath = apiPath; + + this.createAlbum = UrlUtils.join(baseUrl, apiPath.getCreateAlbumPath()); + this.uploadItem = UrlUtils.join(baseUrl, apiPath.getUploadItemPath()); + this.addItemToAlbum = UrlUtils.join(baseUrl, apiPath.getAddItemToAlbumPath()); + } + + public ApiPath getApiPath() { + return apiPath; + } + + public String getCreateAlbum() { + return createAlbum; + } + + public String getUploadItem() { + return uploadItem; + } + + public String getAddItemToAlbum() { + return addItemToAlbum; + } + + public static class ApiPath { + private final String createAlbumPath; + private final String uploadItemPath; + private final String addItemToAlbumPath; + + @JsonCreator + public ApiPath( + @JsonProperty("createAlbum") String createAlbumPath, + @JsonProperty("uploadItem") String uploadItemPath, + @JsonProperty("addItemToAlbum") String addItemToAlbumPath) { + this.createAlbumPath = createAlbumPath; + this.uploadItemPath = uploadItemPath; + this.addItemToAlbumPath = addItemToAlbumPath; + } + + public String getCreateAlbumPath() { + return createAlbumPath; + } + + public String getUploadItemPath() { + return uploadItemPath; + } + + public String getAddItemToAlbumPath() { + return addItemToAlbumPath; + } + } +} diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/models/ServiceConfig.java b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/models/ServiceConfig.java new file mode 100644 index 000000000..55ab1c2a0 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/models/ServiceConfig.java @@ -0,0 +1,86 @@ +/* + * Copyright 2025 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.datatransferproject.datatransfer.synology.models; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import java.util.Map; +import java.util.Objects; + +/** Data model for synology service configuration. */ +public class ServiceConfig { + private final Retry retry; + private final Map services; + + @JsonCreator + public ServiceConfig( + @JsonProperty("retry") Retry retry, @JsonProperty("services") Map services) { + this.retry = Objects.requireNonNull(retry, "retry is required"); + this.services = Objects.requireNonNull(services, "at least 1 service is required"); + } + + public Retry getRetry() { + return retry; + } + + public Map getServices() { + return services; + } + + public T getServiceAs(String name, Class type) { + Service service = services.get(name); + if (type.isInstance(service)) { + return type.cast(service); + } else { + throw new IllegalArgumentException( + "Service '" + name + "' is not of type " + type.getSimpleName()); + } + } + + public static class Retry { + private final int maxAttempts; + + @JsonCreator + public Retry(@JsonProperty("maxAttempts") int maxAttempts) { + this.maxAttempts = maxAttempts; + } + + public int getMaxAttempts() { + return maxAttempts; + } + } + + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") + @JsonSubTypes({ + @JsonSubTypes.Type(value = C2Api.class, name = "c2"), + }) + public static class Service { + private final String baseUrl; + + @JsonCreator + public Service(@JsonProperty("baseUrl") String baseUrl) { + this.baseUrl = Objects.requireNonNull(baseUrl, "baseUrl is required"); + } + + public String getBaseUrl() { + return baseUrl; + } + } +} diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/photos/SynologyPhotosImporter.java b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/photos/SynologyPhotosImporter.java new file mode 100644 index 000000000..423e6e130 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/photos/SynologyPhotosImporter.java @@ -0,0 +1,48 @@ +package org.datatransferproject.datatransfer.synology.photos; + +import java.util.UUID; +import org.datatransferproject.api.launcher.Monitor; +import org.datatransferproject.datatransfer.synology.service.SynologyOAuthTokenManager; +import org.datatransferproject.datatransfer.synology.uploader.SynologyUploader; +import org.datatransferproject.spi.transfer.idempotentexecutor.IdempotentImportExecutor; +import org.datatransferproject.spi.transfer.provider.ImportResult; +import org.datatransferproject.spi.transfer.provider.Importer; +import org.datatransferproject.types.common.models.photos.PhotosContainerResource; +import org.datatransferproject.types.transfer.auth.TokensAndUrlAuthData; + +public class SynologyPhotosImporter + implements Importer { + private final Monitor monitor; + private final SynologyOAuthTokenManager tokenManager; + private final SynologyUploader synologyUploader; + + public SynologyPhotosImporter( + Monitor monitor, SynologyOAuthTokenManager tokenManager, SynologyUploader synologyUploader) { + this.monitor = monitor; + this.tokenManager = tokenManager; + this.synologyUploader = synologyUploader; + + monitor.info(() -> "Creating SynologyPhotosImporter"); + } + + @Override + public ImportResult importItem( + UUID jobId, + IdempotentImportExecutor idempotentExecutor, + TokensAndUrlAuthData authData, + PhotosContainerResource data) + throws Exception { + monitor.info(() -> "==== [SynologyImporter] SynologyPhotosImporter starts importing data ===="); + tokenManager.addAuthDataIfNotExist(jobId, authData); + + try { + synologyUploader.importAlbums(data.getAlbums(), jobId); + synologyUploader.importPhotos(data.getPhotos(), jobId); + } catch (Exception e) { + monitor.severe(() -> "[SynologyImporter] SynologyPhotosImporter failed to import data:" + e); + return new ImportResult(e); + } + + return ImportResult.OK; + } +} diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/service/SynologyDTPService.java b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/service/SynologyDTPService.java new file mode 100644 index 000000000..5dc03f0c8 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/service/SynologyDTPService.java @@ -0,0 +1,284 @@ +/* + * Copyright 2025 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.datatransferproject.datatransfer.synology.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; +import com.google.common.io.ByteStreams; +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Date; +import java.util.Map; +import java.util.UUID; +import okhttp3.FormBody; +import okhttp3.MediaType; +import okhttp3.MultipartBody; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import org.datatransferproject.api.launcher.Monitor; +import org.datatransferproject.datatransfer.synology.exceptions.SynologyHttpException; +import org.datatransferproject.datatransfer.synology.exceptions.SynologyImportException; +import org.datatransferproject.datatransfer.synology.exceptions.SynologyMaxRetriesExceededException; +import org.datatransferproject.datatransfer.synology.models.C2Api; +import org.datatransferproject.datatransfer.synology.models.ServiceConfig; +import org.datatransferproject.datatransfer.synology.utils.ServiceConfigParser; +import org.datatransferproject.spi.cloud.storage.JobStore; +import org.datatransferproject.types.common.models.media.MediaAlbum; +import org.datatransferproject.types.common.models.photos.PhotoModel; +import org.datatransferproject.types.common.models.videos.VideoModel; +import org.datatransferproject.types.transfer.serviceconfig.TransferServiceConfig; + +/** Service for handling Synology DTP operations. */ +public class SynologyDTPService { + private final Monitor monitor; + private final String exportingService; + private final ObjectMapper objectMapper; + private final OkHttpClient client; + private final JobStore jobStore; + private final SynologyOAuthTokenManager tokenManager; + C2Api c2Api; + ServiceConfig.Retry retryConfig; + + /** + * Constructs a new {@code SynologyDTPService} instance. + * + * @param monitor the monitor + * @param exportingService the exporting service + * @param jobStore the job store + * @param tokenManager the token manager + * @param client the HTTP client + */ + public SynologyDTPService( + Monitor monitor, + TransferServiceConfig transferServiceConfig, + String exportingService, + JobStore jobStore, + SynologyOAuthTokenManager tokenManager, + OkHttpClient client) { + ServiceConfig serviceConfig = ServiceConfigParser.parse(transferServiceConfig); + this.c2Api = serviceConfig.getServiceAs("c2", C2Api.class); + this.retryConfig = serviceConfig.getRetry(); + + this.monitor = monitor; + this.exportingService = exportingService; + this.objectMapper = new ObjectMapper(); + this.jobStore = jobStore; + this.tokenManager = tokenManager; + this.client = client; + } + + /** + * getInputStream + * + * @param url + * @return an InputStream Instance + * @throws IOException + */ + protected InputStream getInputStream(String url) throws IOException { + return new URL(url).openStream(); + } + + /** + * Creates album. + * + * @param album the album + * @param jobId the job ID + * @return a map of shape {"data": {"album_id": }} + */ + public Map createAlbum(MediaAlbum album, UUID jobId) { + FormBody.Builder builder = new FormBody.Builder().add("title", album.getName()); + builder.add("album_id", album.getId()); + builder.add("job_id", jobId.toString()); + builder.add("service", exportingService); + monitor.info(() -> "[SynologyImporter] Creating album", album.getName(), jobId); + + return (Map) + sendPostRequest(c2Api.getCreateAlbum(), builder.build(), jobId).get("data"); + } + + /** + * Creates photo. + * + * @param photo the photo + * @param jobId the job ID + * @return a map of shape {"data": {"item_id": }} + */ + public Map createPhoto(PhotoModel photo, UUID jobId) { + byte[] imageBytes; + try { + InputStream inputStream = null; + if (photo.isInTempStore()) { + inputStream = jobStore.getStream(jobId, photo.getFetchableUrl()).getStream(); + } else if (photo.getFetchableUrl() != null) { + inputStream = getInputStream(photo.getFetchableUrl()); + } else { + monitor.severe(() -> "[SynologyImporter] Can't get inputStream for a photo"); + return null; + } + imageBytes = ByteStreams.toByteArray(inputStream); + } catch (MalformedURLException e) { + throw new SynologyImportException("Failed to create url for photo", e); + } catch (IOException e) { + throw new SynologyImportException("Failed to create input stream for photo", e); + } + + RequestBody fileBody = RequestBody.create(MediaType.parse(photo.getMimeType()), imageBytes); + + MultipartBody.Builder builder = + new MultipartBody.Builder() + .setType(MultipartBody.FORM) + .addFormDataPart("file", photo.getTitle(), fileBody) + .addFormDataPart("item_id", photo.getDataId()) + .addFormDataPart("title", photo.getTitle()) + .addFormDataPart("job_id", jobId.toString()) + .addFormDataPart("service", exportingService); + + String imageDescription = photo.getDescription(); + if (!Strings.isNullOrEmpty(imageDescription)) { + builder.addFormDataPart("description", imageDescription); + } + + @SuppressWarnings("unchecked") + Map responseData = + (Map) + sendPostRequest(c2Api.getUploadItem(), builder.build(), jobId).get("data"); + return responseData; + } + + /** + * Creates video. + * + * @param video the video + * @param jobId the job ID + * @return a map of shape {"data": {"item_id": }} + */ + public Map createVideo(VideoModel video, UUID jobId) { + byte[] videoBytes; + try { + InputStream inputStream = null; + if (video.isInTempStore()) { + inputStream = jobStore.getStream(jobId, video.getFetchableUrl()).getStream(); + } else if (video.getFetchableUrl() != null) { + inputStream = getInputStream(video.getFetchableUrl()); + } else { + monitor.severe(() -> "[SynologyImporter] Can't get inputStream for a video"); + return null; + } + + videoBytes = ByteStreams.toByteArray(inputStream); + } catch (MalformedURLException e) { + throw new SynologyImportException("Failed to create url for video", e); + } catch (IOException e) { + throw new SynologyImportException("Failed to create input stream for video", e); + } + + RequestBody fileBody = RequestBody.create(MediaType.parse(video.getMimeType()), videoBytes); + MultipartBody.Builder builder = + new MultipartBody.Builder() + .setType(MultipartBody.FORM) + .addFormDataPart("file", video.getName(), fileBody) + .addFormDataPart("item_id", video.getDataId()) + .addFormDataPart("title", video.getName()) + .addFormDataPart("job_id", jobId.toString()) + .addFormDataPart("service", exportingService); + + String imageDescription = video.getDescription(); + if (!Strings.isNullOrEmpty(imageDescription)) { + builder.addFormDataPart("description", imageDescription); + } + + @SuppressWarnings("unchecked") + Map responseData = + (Map) + sendPostRequest(c2Api.getUploadItem(), builder.build(), jobId).get("data"); + return responseData; + } + + /** + * Adds item to album. + * + * @param albumId the album ID + * @param itemId the item ID + * @return a map of shape {"success": } + */ + public Map addItemToAlbum(String albumId, String itemId, UUID jobId) { + FormBody.Builder builder = + new FormBody.Builder() + .add("job_id", jobId.toString()) + .add("service", exportingService) + .add("album_id", albumId) + .add("item_id", itemId); + return sendPostRequest(c2Api.getAddItemToAlbum(), builder.build(), jobId); + } + + @VisibleForTesting + protected Map sendPostRequest(String url, RequestBody body, UUID jobId) { + boolean triedRefreshToken = false; + Request.Builder requestBuilder = new Request.Builder().url(url).post(body); + + Exception lastException = null; + for (int retry = retryConfig.getMaxAttempts(); retry > 0; retry--) { + Response response = null; + try { + requestBuilder.header("Authorization", "Bearer " + tokenManager.getAccessToken(jobId)); + response = client.newCall(requestBuilder.build()).execute(); + if (!response.isSuccessful()) { + int code = response.code(); + if (code == 401 && !triedRefreshToken) { + triedRefreshToken = true; + if (tokenManager.refreshToken(jobId, client, objectMapper)) { + continue; + } + } + throw new SynologyHttpException("Synology request failed", code, response.message()); + } + } catch (Exception e) { + monitor.severe( + () -> "[SynologyImporter] Failed to send post request,", "url:", url, "exception:", e); + if (response != null) { + response.close(); + } + lastException = e; + continue; + } + + try { + assert response.body() != null; + String bodyString = response.body().string(); + Map responseData = + (Map) objectMapper.readValue(bodyString, Map.class); + return responseData; + } catch (Exception e) { + monitor.severe( + () -> "[SynologyImporter] Failed to parse response data,", + "url:", + url, + "exception:", + e); + lastException = e; + } + } + throw new SynologyMaxRetriesExceededException( + "Failed to send POST request", retryConfig.getMaxAttempts(), lastException); + } +} diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/service/SynologyOAuthTokenManager.java b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/service/SynologyOAuthTokenManager.java new file mode 100644 index 000000000..6bba86966 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/service/SynologyOAuthTokenManager.java @@ -0,0 +1,129 @@ +/* + * Copyright 2023 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.datatransferproject.datatransfer.synology.service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import okhttp3.FormBody; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import org.datatransferproject.api.launcher.Monitor; +import org.datatransferproject.datatransfer.synology.exceptions.SynologyImportException; +import org.datatransferproject.types.transfer.auth.AppCredentials; +import org.datatransferproject.types.transfer.auth.TokensAndUrlAuthData; + +/** Manages the OAuth tokens from Synology Account. */ +public class SynologyOAuthTokenManager { + private final Map authMap = new HashMap<>(); + private final AppCredentials appCredentials; + private final Monitor monitor; + + public SynologyOAuthTokenManager(AppCredentials appCredentials, Monitor monitor) { + this.appCredentials = appCredentials; + this.monitor = monitor; + } + + public String getAccessToken(UUID jobId) { + if (!authMap.containsKey(jobId)) { + throw new SynologyImportException("No auth data found for job: " + jobId); + } + return authMap.get(jobId).getAccessToken(); + } + + public void addAuthDataIfNotExist(UUID jobId, TokensAndUrlAuthData authData) { + authMap.putIfAbsent(jobId, authData); + } + + public boolean refreshToken(UUID jobId, OkHttpClient client, ObjectMapper objectMapper) { + monitor.info( + () -> "=== [SynologyImporter] Refresh Synology Account token for job:", jobId, "==="); + if (!authMap.containsKey(jobId)) { + monitor.severe(() -> "[SynologyImporter] No auth data found for job:", jobId); + return false; + } + + TokensAndUrlAuthData authData = authMap.get(jobId); + Request request = + new Request.Builder() + .url(authData.getTokenServerEncodedUrl()) + .addHeader("Accept", "application/json") + .post( + new FormBody.Builder() + .add("client_id", appCredentials.getKey()) + .add("client_secret", appCredentials.getSecret()) + .add("refresh_token", authData.getRefreshToken()) + .add("grant_type", "refresh_token") + .build()) + .build(); + Response response; + try { + response = client.newCall(request).execute(); + } catch (IOException e) { + monitor.severe( + () -> "[SynologyImporter] Failed to send refresh token request for job:", + jobId, + "with exception:", + e); + return false; + } + + String responseBody; + try { + responseBody = response.body().string(); + if (!response.isSuccessful()) { + monitor.severe( + () -> "[SynologyImporter] Failed to refresh token for job:", + jobId, + "with failed response:", + responseBody); + return false; + } + } catch (IOException e) { + monitor.severe( + () -> "[SynologyImporter] Failed to stringify response body, jobId:", + jobId, + "with exception:", + e); + return false; + } + + try { + Map responseData = objectMapper.readValue(responseBody, Map.class); + String accessToken = (String) responseData.get("access_token"); + String refreshToken = (String) responseData.get("refresh_token"); + authData = + new TokensAndUrlAuthData(accessToken, refreshToken, authData.getTokenServerEncodedUrl()); + authMap.replace(jobId, authData); + } catch (JsonProcessingException e) { + monitor.severe( + () -> "[SynologyImporter] Failed to parse response for job:", + jobId, + "with exception:", + e); + return false; + } + + monitor.info( + () -> "=== [SynologyImporter] Successfully refreshed token for job:", jobId, "==="); + return true; + } +} diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/uploader/SynologyUploader.java b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/uploader/SynologyUploader.java new file mode 100644 index 000000000..111936d80 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/uploader/SynologyUploader.java @@ -0,0 +1,253 @@ +/* + * Copyright 2025 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.datatransferproject.datatransfer.synology.uploader; + +import java.util.Collection; +import java.util.Map; +import java.util.UUID; +import java.util.function.BiFunction; +import java.util.function.Function; +import org.datatransferproject.api.launcher.Monitor; +import org.datatransferproject.datatransfer.synology.constant.SynologyConstant; +import org.datatransferproject.datatransfer.synology.exceptions.SynologyImportException; +import org.datatransferproject.datatransfer.synology.service.SynologyDTPService; +import org.datatransferproject.datatransfer.synology.utils.SynologyMediaAlbumBinder; +import org.datatransferproject.spi.transfer.idempotentexecutor.IdempotentImportExecutor; +import org.datatransferproject.types.common.DownloadableItem; +import org.datatransferproject.types.common.ImportableItem; +import org.datatransferproject.types.common.models.media.MediaAlbum; +import org.datatransferproject.types.common.models.photos.PhotoAlbum; +import org.datatransferproject.types.common.models.photos.PhotoModel; +import org.datatransferproject.types.common.models.videos.VideoAlbum; +import org.datatransferproject.types.common.models.videos.VideoModel; + +/** Handles the import of albums, photos, and videos to Synology. Business logic is handled here. */ +public class SynologyUploader { + private final IdempotentImportExecutor idempotentExecutor; + private final Monitor monitor; + private final SynologyDTPService synologyDTPService; + + // need a mapper to map PhotoModel.albumId to newAlbumId from SynologyDTPService.createAlbum + private final SynologyMediaAlbumBinder synologyMediaAlbumBinder; + + /** + * Constructs a new {@code SynologyUploader} instance. + * + * @param idempotentExecutor the idempotent executor + * @param monitor the monitor + * @param synologyDTPService the Synology DTP service + */ + public SynologyUploader( + IdempotentImportExecutor idempotentExecutor, + Monitor monitor, + SynologyDTPService synologyDTPService) { + this.idempotentExecutor = idempotentExecutor; + this.monitor = monitor; + this.synologyDTPService = synologyDTPService; + + this.synologyMediaAlbumBinder = + new SynologyMediaAlbumBinder(this::addItemToAlbum, monitor); + } + + /** + * Imports albums. + * + * @param albums the albums + * @param jobId the job ID + */ + public void importAlbums(Collection albums, UUID jobId) { + if (albums.isEmpty()) { + return; + } + monitor.info(() -> "[SynologyImporter] starts importing albums", jobId); + albums.forEach( + album -> { + try { + MediaAlbum mediaAlbum; + if (album instanceof MediaAlbum) { + mediaAlbum = (MediaAlbum) album; + } else if (album instanceof PhotoAlbum) { + mediaAlbum = MediaAlbum.photoToMediaAlbum(((PhotoAlbum) album)); + } else if (album instanceof VideoAlbum) { + mediaAlbum = MediaAlbum.videoToMediaAlbum(((VideoAlbum) album)); + } else { + throw new SynologyImportException("Unexpected album type"); + } + String newAlbumId = + importItemWithCache( + mediaAlbum, + jobId, + "album_id", + synologyDTPService::createAlbum, + MediaAlbum::getId, + MediaAlbum::getName); + synologyMediaAlbumBinder.whenAlbumReady(mediaAlbum.getId(), newAlbumId, jobId); + } catch (Exception e) { + monitor.severe(e::toString, jobId); + throw new SynologyImportException("Failed to import albums", e); + } + }); + monitor.info(() -> "[SynologyImporter] imported albums successfully", jobId); + } + + /** + * Imports photos. + * + * @param photos the photos + * @param jobId the job ID + */ + public void importPhotos(Collection photos, UUID jobId) { + if (photos.isEmpty()) { + return; + } + monitor.info(() -> "[SynologyImporter] starts importing photos", jobId); + photos.forEach( + photo -> { + try { + String newPhotoId = + importDownloadableItemWithCache( + photo, + jobId, + PhotoModel::getAlbumId, + synologyDTPService::createPhoto, + PhotoModel::getDataId, + PhotoModel::getName); + synologyMediaAlbumBinder.put(photo.getAlbumId(), newPhotoId, jobId); + } catch (Exception e) { + monitor.severe(e::toString, jobId); + throw new SynologyImportException("Failed to import photos", e); + } + }); + monitor.info(() -> "[SynologyImporter] imported photos successfully", jobId); + } + + /** + * Imports videos. + * + * @param videos the videos + * @param jobId the job ID + */ + public void importVideos(Collection videos, UUID jobId) { + if (videos.isEmpty()) { + return; + } + monitor.info(() -> "[SynologyImporter] starts importing videos", jobId); + videos.forEach( + video -> { + try { + String newVideoId = + importDownloadableItemWithCache( + video, + jobId, + VideoModel::getAlbumId, + synologyDTPService::createVideo, + VideoModel::getDataId, + VideoModel::getName); + synologyMediaAlbumBinder.put(video.getAlbumId(), newVideoId, jobId); + } catch (Exception e) { + monitor.severe(e::toString, jobId); + throw new SynologyImportException("Failed to import videos", e); + } + }); + monitor.info(() -> "[SynologyImporter] imported videos successfully", jobId); + } + ; + + /** + * Imports a item + * + * @param item the item + * @return the item ID + */ + private String importDownloadableItemWithCache( + T item, + UUID jobId, + Function albumIdFunction, + BiFunction> createFunction, + Function dataIdFunction, + Function nameFunction) + throws SynologyImportException { + String newItemId = null; + try { + newItemId = + importItemWithCache(item, jobId, "item_id", createFunction, dataIdFunction, nameFunction); + } catch (Exception e) { + throw new SynologyImportException( + String.format( + "Failed to import item [%s], name [%s]", + dataIdFunction.apply(item), nameFunction.apply(item)), + e); + } + + return newItemId; + } + + private String importItemWithCache( + T item, + UUID jobId, + String idField, + BiFunction> createFunction, + Function dataIdFunction, + Function nameFunction) + throws Exception { + return idempotentExecutor.executeAndSwallowIOExceptions( + dataIdFunction.apply(item), + nameFunction.apply(item), + () -> { + Map createResult = createFunction.apply(item, jobId); + String itemId = createResult.get(idField).toString(); + monitor.info( + () -> "[SynologyImporter] item uploaded successfully", + jobId, + "dataId:", + dataIdFunction.apply(item), + "name:", + nameFunction.apply(item)); + return itemId; + }); + } + + private void addItemToAlbum(String albumId, String itemId, UUID jobId) { + try { + Boolean createResult = + idempotentExecutor.executeAndSwallowIOExceptions( + String.format(SynologyConstant.ALBUM_ITEM_ID_FORMAT, albumId, itemId), + itemId, + () -> + (Boolean) + synologyDTPService.addItemToAlbum(albumId, itemId, jobId).get("success")); + if (Boolean.FALSE.equals(createResult)) { + throw new SynologyImportException( + String.format( + "Unsuccessful result from adding item [%s] to album [%s]", itemId, albumId)); + } + } catch (SynologyImportException e) { + throw e; + } catch (Exception e) { + throw new SynologyImportException( + String.format("Failed to add item [%s] to album [%s]", itemId, albumId), e); + } + monitor.info( + () -> "[SynologyImporter] added item to album successfully", + jobId, + "albumId:", + albumId, + "itemId:", + itemId); + } +} diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/utils/OnAlbumReadyCallback.java b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/utils/OnAlbumReadyCallback.java new file mode 100644 index 000000000..16cb90034 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/utils/OnAlbumReadyCallback.java @@ -0,0 +1,30 @@ +/* + * Copyright 2025 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.datatransferproject.datatransfer.synology.utils; + +import java.util.UUID; + +/** + * Callback interface for notifying when an album is ready. + * + * @param the album key type + */ +@FunctionalInterface +public interface OnAlbumReadyCallback { + void accept(K newAlbumKey, K newItemKey, UUID jobId); +} diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/utils/ServiceConfigParser.java b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/utils/ServiceConfigParser.java new file mode 100644 index 000000000..c5bef0b78 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/utils/ServiceConfigParser.java @@ -0,0 +1,47 @@ +/* + * Copyright 2025 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.datatransferproject.datatransfer.synology.utils; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Optional; +import org.datatransferproject.datatransfer.synology.models.ServiceConfig; +import org.datatransferproject.types.transfer.serviceconfig.TransferServiceConfig; + +public class ServiceConfigParser { + private static final ObjectMapper mapper = new ObjectMapper(); + + public static ServiceConfig parse(TransferServiceConfig config) { + return parse(config.getServiceConfig()); + } + + public static ServiceConfig parse(JsonNode node) { + try { + return mapper.treeToValue(node, ServiceConfig.class); + } catch (JsonProcessingException e) { + throw new RuntimeException("Failed to parse serviceConfig into ServiceConfig object", e); + } + } + + public static ServiceConfig parse(Optional maybeNode) { + return maybeNode + .map(ServiceConfigParser::parse) + .orElseThrow(() -> new IllegalArgumentException("ServiceConfig is required")); + } +} diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/utils/SynologyMediaAlbumBinder.java b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/utils/SynologyMediaAlbumBinder.java new file mode 100644 index 000000000..b1c7bcb77 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/utils/SynologyMediaAlbumBinder.java @@ -0,0 +1,110 @@ +/* + * Copyright 2025 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.datatransferproject.datatransfer.synology.utils; + +import java.util.Queue; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ConcurrentMap; +import javax.annotation.Nullable; +import org.datatransferproject.api.launcher.Monitor; + +/** + * Binds media items to albums, ensuring that items are only added to albums that are ready. NOTE: + * This class is thread-safe. + * + * @param the album key type + */ +public class SynologyMediaAlbumBinder { + private final OnAlbumReadyCallback onAlbumReady; + private final Monitor monitor; + + private final ConcurrentMap readyAlbumMap = new ConcurrentHashMap<>(); + private final ConcurrentMap> pendingItemMap = new ConcurrentHashMap<>(); + + public SynologyMediaAlbumBinder(OnAlbumReadyCallback onAlbumReady, Monitor monitor) { + this.onAlbumReady = onAlbumReady; + this.monitor = monitor; + } + + public void whenAlbumReady(@Nullable K albumKey, @Nullable K newAlbumKey, @Nullable UUID jobId) { + if (albumKey == null || newAlbumKey == null) { + return; + } + readyAlbumMap.put(albumKey, newAlbumKey); + monitor.debug( + () -> "[SynologyMediaBinder] album READY", + "albumKey:", + albumKey, + "newAlbumKey:", + newAlbumKey, + jobId); + consumePendingItems(albumKey, jobId); + } + + public void put(@Nullable K albumKey, @Nullable K newItemKey, @Nullable UUID jobId) { + if (albumKey == null || newItemKey == null) { + return; + } + // unused variable '_' is not allowed in java 11 so '__' instead + readyAlbumMap.compute( + albumKey, + (__, value) -> { + if (value == null) { + pendingItemMap + .computeIfAbsent(albumKey, k -> new ConcurrentLinkedQueue<>()) + .add(newItemKey); + monitor.debug( + () -> "[SynologyMediaBinder] item PENDING", + "albumKey:", + albumKey, + "newItemKey:", + newItemKey, + jobId); + } else { + onAlbumReady.accept(value, newItemKey, jobId); + monitor.debug( + () -> "[SynologyMediaBinder] item PROCESSED", + "albumKey:", + albumKey, + "newItemKey:", + newItemKey, + jobId); + } + return value; + }); + } + + private void consumePendingItems(K albumKey, UUID jobId) { + Queue pendingNewItemKeys = pendingItemMap.remove(albumKey); + if (pendingNewItemKeys != null) { + K album = readyAlbumMap.get(albumKey); + for (K newItemKey : pendingNewItemKeys) { + onAlbumReady.accept(album, newItemKey, jobId); + } + monitor.debug( + () -> "[SynologyMediaBinder] pending items CONSUMING", + "albumKey:", + albumKey, + "pendingItemCount:", + pendingNewItemKeys.size(), + jobId); + } + } +} diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/utils/UrlUtils.java b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/utils/UrlUtils.java new file mode 100644 index 000000000..5ca9ebe11 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/utils/UrlUtils.java @@ -0,0 +1,39 @@ +/* + * Copyright 2025 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.datatransferproject.datatransfer.synology.utils; + +import javax.annotation.Nonnull; + +public class UrlUtils { + public static String join(@Nonnull String baseUrl, @Nonnull String path) { + if (baseUrl.isEmpty()) { + throw new IllegalArgumentException("Base URL cannot be empty"); + } + + boolean baseEndsWithSlash = baseUrl.endsWith("/"); + boolean pathStartsWithSlash = path.startsWith("/"); + + if (baseEndsWithSlash && pathStartsWithSlash) { + return baseUrl + path.substring(1); + } else if (!baseEndsWithSlash && !pathStartsWithSlash) { + return baseUrl + "/" + path; + } else { + return baseUrl + path; + } + } +} diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/videos/SynologyVideosImporter.java b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/videos/SynologyVideosImporter.java new file mode 100644 index 000000000..9fee36cc7 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/main/java/org/datatransferproject/datatransfer/synology/videos/SynologyVideosImporter.java @@ -0,0 +1,48 @@ +package org.datatransferproject.datatransfer.synology.videos; + +import java.util.UUID; +import org.datatransferproject.api.launcher.Monitor; +import org.datatransferproject.datatransfer.synology.service.SynologyOAuthTokenManager; +import org.datatransferproject.datatransfer.synology.uploader.SynologyUploader; +import org.datatransferproject.spi.transfer.idempotentexecutor.IdempotentImportExecutor; +import org.datatransferproject.spi.transfer.provider.ImportResult; +import org.datatransferproject.spi.transfer.provider.Importer; +import org.datatransferproject.types.common.models.videos.VideosContainerResource; +import org.datatransferproject.types.transfer.auth.TokensAndUrlAuthData; + +public class SynologyVideosImporter + implements Importer { + private final Monitor monitor; + private final SynologyOAuthTokenManager tokenManager; + private final SynologyUploader synologyUploader; + + public SynologyVideosImporter( + Monitor monitor, SynologyOAuthTokenManager tokenManager, SynologyUploader synologyUploader) { + this.monitor = monitor; + this.tokenManager = tokenManager; + this.synologyUploader = synologyUploader; + + monitor.info(() -> "Creating SynologyVideosImporter"); + } + + @Override + public ImportResult importItem( + UUID jobId, + IdempotentImportExecutor idempotentExecutor, + TokensAndUrlAuthData authData, + VideosContainerResource data) + throws Exception { + monitor.info(() -> "==== [SynologyImporter] SynologyVideosImporter starts importing data ===="); + tokenManager.addAuthDataIfNotExist(jobId, authData); + + try { + synologyUploader.importAlbums(data.getAlbums(), jobId); + synologyUploader.importVideos(data.getVideos(), jobId); + } catch (Exception e) { + monitor.severe(() -> "[SynologyImporter] SynologyVideosImporter failed to import data:" + e); + return new ImportResult(e); + } + + return ImportResult.OK; + } +} diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/main/resources/META-INF/services/org.datatransferproject.spi.transfer.extension.TransferExtension b/extensions/data-transfer/portability-data-transfer-synology/src/main/resources/META-INF/services/org.datatransferproject.spi.transfer.extension.TransferExtension new file mode 100644 index 000000000..4490ea0c6 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/main/resources/META-INF/services/org.datatransferproject.spi.transfer.extension.TransferExtension @@ -0,0 +1 @@ +org.datatransferproject.datatransfer.synology.SynologyTransferExtension diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/main/resources/config/synology.yaml b/extensions/data-transfer/portability-data-transfer-synology/src/main/resources/config/synology.yaml new file mode 100644 index 000000000..07b57234b --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/main/resources/config/synology.yaml @@ -0,0 +1,13 @@ +# TODO(ehung): confirm synology sa rate limit +perUserRateLimit: 1 +serviceConfig: + retry: + maxAttempts: 3 + services: + c2: + type: "c2" + baseUrl: "https://sa.dtp.us.c2.synology.com" + apiPath: + createAlbum: "/import/album" + uploadItem: "/import/item" + addItemToAlbum: "/import/album/item" diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/test/java/org/datatransferproject/datatransfer/synology/models/ServiceConfigTest.java b/extensions/data-transfer/portability-data-transfer-synology/src/test/java/org/datatransferproject/datatransfer/synology/models/ServiceConfigTest.java new file mode 100644 index 000000000..4a598de30 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/test/java/org/datatransferproject/datatransfer/synology/models/ServiceConfigTest.java @@ -0,0 +1,69 @@ +/* + * Copyright 2025 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.datatransferproject.datatransfer.synology.models; + +import static org.junit.jupiter.api.Assertions.*; + +import java.util.Map; +import org.datatransferproject.datatransfer.synology.models.ServiceConfig.Service; +import org.junit.jupiter.api.Test; + +public class ServiceConfigTest { + @Test + public void testServiceConfig() { + ServiceConfig.Retry retryConfig = new ServiceConfig.Retry(3); + Map services = Map.of("synologyService", new Service("http://example.com")); + + ServiceConfig config = new ServiceConfig(retryConfig, services); + + Service service = config.getServiceAs("synologyService", Service.class); + assertNotNull(service); + assertEquals("http://example.com", service.getBaseUrl()); + + ServiceConfig.Retry retry = config.getRetry(); + assertNotNull(retry); + assertEquals(3, retry.getMaxAttempts()); + + Map serviceMap = config.getServices(); + assertNotNull(serviceMap); + assertEquals(1, serviceMap.size()); + assertTrue(serviceMap.containsKey("synologyService")); + } + + @Test + public void testServiceConfigWithCustomService() { + ServiceConfig.Retry retryConfig = new ServiceConfig.Retry(3); + Map services = Map.of("customService", new CustomService("http://custom.com")); + + ServiceConfig config = new ServiceConfig(retryConfig, services); + + CustomService service = config.getServiceAs("customService", CustomService.class); + assertNotNull(service); + assertEquals("http://custom.com", service.getBaseUrl()); + + ServiceConfig.Retry retry = config.getRetry(); + assertNotNull(retry); + assertEquals(3, retry.getMaxAttempts()); + } + + public static class CustomService extends Service { + public CustomService(String baseUrl) { + super(baseUrl); + } + } +} diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/test/java/org/datatransferproject/datatransfer/synology/service/SynologyDTPServiceTest.java b/extensions/data-transfer/portability-data-transfer-synology/src/test/java/org/datatransferproject/datatransfer/synology/service/SynologyDTPServiceTest.java new file mode 100644 index 000000000..607781e3a --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/test/java/org/datatransferproject/datatransfer/synology/service/SynologyDTPServiceTest.java @@ -0,0 +1,615 @@ +/* + * Copyright 2025 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.datatransferproject.datatransfer.synology.service; + +import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR; +import static org.apache.http.HttpStatus.SC_UNAUTHORIZED; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.*; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Stream; +import okhttp3.Call; +import okhttp3.FormBody; +import okhttp3.MediaType; +import okhttp3.MultipartBody; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import okhttp3.ResponseBody; +import okio.Buffer; +import org.datatransferproject.api.launcher.Monitor; +import org.datatransferproject.datatransfer.synology.exceptions.SynologyImportException; +import org.datatransferproject.datatransfer.synology.exceptions.SynologyMaxRetriesExceededException; +import org.datatransferproject.datatransfer.synology.utils.TestConfigs; +import org.datatransferproject.spi.cloud.storage.JobStore; +import org.datatransferproject.spi.cloud.storage.TemporaryPerJobDataStore.InputStreamWrapper; +import org.datatransferproject.types.common.DownloadableFile; +import org.datatransferproject.types.common.models.media.MediaAlbum; +import org.datatransferproject.types.common.models.photos.PhotoModel; +import org.datatransferproject.types.common.models.videos.VideoModel; +import org.datatransferproject.types.transfer.serviceconfig.TransferServiceConfig; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestInstance.Lifecycle; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class SynologyDTPServiceTest { + private final String exportingService = "mockService"; + private final UUID jobId = UUID.randomUUID(); + protected SynologyDTPService dtpService; + @Mock protected Monitor monitor; + @Mock protected TransferServiceConfig transferServiceConfig; + @Mock protected JobStore jobStore; + @Mock protected SynologyOAuthTokenManager tokenManager; + @Captor ArgumentCaptor requestBodyCaptor; + @Mock private OkHttpClient client; + + @BeforeEach + public void setUp() { + lenient().when(tokenManager.getAccessToken(jobId)).thenReturn("mockAccessToken"); + lenient() + .when(transferServiceConfig.getServiceConfig()) + .thenReturn(TestConfigs.createServiceConfigJson()); + + dtpService = + new SynologyDTPService( + monitor, transferServiceConfig, exportingService, jobStore, tokenManager, client); + } + + @Nested + public class AddItemToAlbum { + private final String albumId = "testAlbum"; + private final String itemId = "testItem"; + + @Test + public void shouldSendPostRequestWithCorrectFormBody() { + SynologyDTPService spyService = Mockito.spy(dtpService); + + doReturn(Map.of("success", true)).when(spyService).sendPostRequest(anyString(), any(), any()); + Map result = spyService.addItemToAlbum(albumId, itemId, jobId); + + verify(spyService).sendPostRequest(anyString(), requestBodyCaptor.capture(), any()); + assertEquals(result.get("success"), true); + + RequestBody capturedBody = requestBodyCaptor.getValue(); + assertTrue(capturedBody instanceof FormBody); + FormBody formBody = (FormBody) capturedBody; + + Map formBodyMap = + Map.of( + "job_id", jobId.toString(), + "service", exportingService, + "album_id", albumId, + "item_id", itemId); + for (int i = 0; i < formBody.size(); i++) { + assertEquals(formBodyMap.get(formBody.name(i)), formBody.value(i)); + } + } + + @Test + public void shouldThrowExceptionIfSendPostRequestFailed() { + SynologyDTPService spyService = Mockito.spy(dtpService); + + doThrow( + new SynologyMaxRetriesExceededException( + "MockException", TestConfigs.TEST_MAX_ATTEMPTS, new Exception("Failed"))) + .when(spyService) + .sendPostRequest(anyString(), any(), any()); + + assertThrows( + SynologyMaxRetriesExceededException.class, + () -> spyService.addItemToAlbum(albumId, itemId, jobId), + "MockException"); + } + } + + @Nested + public class CreateAlbum { + private final String albumId = "testAlbumId"; + private final String albumName = "testAlbumName"; + private final MediaAlbum album = new MediaAlbum(albumId, albumName, ""); + + @Test + public void shouldSendPostRequestWithCorrectFormBody() { + SynologyDTPService spyService = Mockito.spy(dtpService); + Map dataMap = Map.of("album_id", albumId); + + doReturn(Map.of("success", true, "data", dataMap)) + .when(spyService) + .sendPostRequest(anyString(), any(), any()); + Map result = spyService.createAlbum(album, jobId); + + verify(spyService).sendPostRequest(anyString(), requestBodyCaptor.capture(), any()); + assertEquals(result.get("success"), null); + assertEquals(result.get("album_id"), albumId); + + RequestBody capturedBody = requestBodyCaptor.getValue(); + assertTrue(capturedBody instanceof FormBody); + FormBody formBody = (FormBody) capturedBody; + + Map formBodyMap = + Map.of( + "job_id", jobId.toString(), + "service", exportingService, + "album_id", albumId, + "title", albumName); + for (int i = 0; i < formBody.size(); i++) { + assertEquals(formBodyMap.get(formBody.name(i)), formBody.value(i)); + } + } + + @Test + public void shouldThrowExceptionIfSendPostRequestFailed() { + SynologyDTPService spyService = Mockito.spy(dtpService); + + doThrow( + new SynologyMaxRetriesExceededException( + "MockException", TestConfigs.TEST_MAX_ATTEMPTS, new Exception("Failed"))) + .when(spyService) + .sendPostRequest(anyString(), any(), any()); + + assertThrows( + SynologyMaxRetriesExceededException.class, + () -> spyService.createAlbum(album, jobId), + "MockException"); + } + } + + @Nested + @TestInstance(Lifecycle.PER_CLASS) + public class CreateMedia { + private final String itemName = "itemName"; + private final String fetchUrl = "https://example.com"; + private final String description = "desc"; + private final String itemId = "itemId"; + + public Stream provideMediaObjectsInTempStore() { + return Stream.of( + new PhotoModel(itemName, fetchUrl, description, "mediaType", itemId, null, true), + new VideoModel(itemName, fetchUrl, description, "format", itemId, null, true, null)); + } + + public Stream provideMediaObjectsWithoutDescriptionInTempStore() { + return Stream.of( + new PhotoModel(itemName, fetchUrl, null, "mediaType", itemId, null, true), + new VideoModel(itemName, fetchUrl, null, "format", itemId, null, true, null)); + } + + public Stream provideMediaObjectsNotInTempStore() { + return Stream.of( + new PhotoModel(itemName, fetchUrl, description, "mediaType", itemId, null, false), + new VideoModel(itemName, fetchUrl, description, "format", itemId, null, false, null)); + } + + @ParameterizedTest(name = "shouldGetStreamFromJobStoreIfMediaItemIsInTempStore [{index}] {0}") + @MethodSource("provideMediaObjectsInTempStore") + public void shouldGetStreamFromJobStoreIfMediaItemIsInTempStore(DownloadableFile item) + throws IOException { + byte[] mockImage = new byte[] {1, 2, 3}; + InputStream mockInputStream = new ByteArrayInputStream(mockImage); + InputStreamWrapper streamWrapper = mock(InputStreamWrapper.class); + SynologyDTPService spyService = Mockito.spy(dtpService); + + when(jobStore.getStream(jobId, fetchUrl)).thenReturn(streamWrapper); + when(streamWrapper.getStream()).thenReturn(mockInputStream); + doReturn(mock(Map.class)).when(spyService).sendPostRequest(anyString(), any(), any()); + + if (item instanceof PhotoModel) { + spyService.createPhoto((PhotoModel) item, jobId); + } else if (item instanceof VideoModel) { + spyService.createVideo((VideoModel) item, jobId); + } + + verify(spyService).sendPostRequest(anyString(), any(), any()); + verify(streamWrapper).getStream(); + verify(spyService, never()).getInputStream(fetchUrl); + } + + @ParameterizedTest(name = "shouldGetStreamFromUrlIfPhotoIsNotInTempStore [{index}] {0}") + @MethodSource("provideMediaObjectsNotInTempStore") + public void shouldGetStreamFromUrlIfPhotoIsNotInTempStore(DownloadableFile item) + throws IOException { + byte[] mockImage = new byte[] {1, 2, 3}; + SynologyDTPService spyService = Mockito.spy(dtpService); + InputStream mockInputStream = new ByteArrayInputStream(mockImage); + + doReturn(mockInputStream).when(spyService).getInputStream(fetchUrl); + doReturn(mock(Map.class)).when(spyService).sendPostRequest(anyString(), any(), any()); + + if (item instanceof PhotoModel) { + spyService.createPhoto((PhotoModel) item, jobId); + } else if (item instanceof VideoModel) { + spyService.createVideo((VideoModel) item, jobId); + } + + verify(spyService).sendPostRequest(anyString(), any(), any()); + verify(spyService).getInputStream(fetchUrl); + verifyNoInteractions(jobStore); + } + + // VideoModel.contentUrl can't be null + @Test + public void shouldReturnNullWhenPhotoIsNotInTempStoreAndHasNoUrl() { + PhotoModel photo = + new PhotoModel(itemName, null, description, "mediaType", itemId, null, false); + SynologyDTPService spyService = Mockito.spy(dtpService); + assertEquals(null, spyService.createPhoto((PhotoModel) photo, jobId)); + verify(spyService, never()).sendPostRequest(anyString(), any(), any()); + } + + @ParameterizedTest(name = "shouldThrowExceptionIfNewURLFailed [{index}] {0}") + @MethodSource("provideMediaObjectsNotInTempStore") + public void shouldThrowExceptionIfNewURLFailed(DownloadableFile item) throws IOException { + SynologyDTPService spyService = Mockito.spy(dtpService); + doThrow(new MalformedURLException("Failed to create url for photo")) + .when(spyService) + .getInputStream(fetchUrl); + + if (item instanceof PhotoModel) { + assertThrows( + SynologyImportException.class, + () -> spyService.createPhoto((PhotoModel) item, jobId), + "Failed to create url for photo"); + } else if (item instanceof VideoModel) { + assertThrows( + SynologyImportException.class, + () -> spyService.createVideo((VideoModel) item, jobId), + "Failed to create url for video"); + } + + verify(spyService, never()).sendPostRequest(anyString(), any(), any()); + } + + @ParameterizedTest(name = "shouldThrowExceptionIfFailedToGetStream [{index}] {0}") + @MethodSource("provideMediaObjectsInTempStore") + public void shouldThrowExceptionIfFailedToGetStream(DownloadableFile item) + throws IOException, SynologyMaxRetriesExceededException { + SynologyDTPService spyService = Mockito.spy(dtpService); + when(jobStore.getStream(jobId, fetchUrl)).thenThrow(new IOException("Failed to get stream")); + + if (item instanceof PhotoModel) { + assertThrows( + SynologyImportException.class, + () -> spyService.createPhoto((PhotoModel) item, jobId), + "Failed to create input stream for photo"); + } else if (item instanceof VideoModel) { + assertThrows( + SynologyImportException.class, + () -> spyService.createVideo((VideoModel) item, jobId), + "Failed to create input stream for video"); + } + verify(spyService, never()).sendPostRequest(anyString(), any(), any()); + } + + @ParameterizedTest( + name = "shouldSendPostRequestWithCorrectFormBodyWithDescription [{index}] {0}") + @MethodSource("provideMediaObjectsInTempStore") + public void shouldSendPostRequestWithCorrectFormBodyWithDescription(DownloadableFile item) + throws IOException { + byte[] mockImage = new byte[] {1, 2, 3}; + InputStream mockInputStream = new ByteArrayInputStream(mockImage); + InputStreamWrapper streamWrapper = mock(InputStreamWrapper.class); + SynologyDTPService spyService = Mockito.spy(dtpService); + Map dataMap = Map.of("item_id", itemId); + + when(jobStore.getStream(jobId, fetchUrl)).thenReturn(streamWrapper); + when(streamWrapper.getStream()).thenReturn(mockInputStream); + doReturn(Map.of("success", true, "data", dataMap)) + .when(spyService) + .sendPostRequest(anyString(), any(), any()); + + Map result = new HashMap(); + if (item instanceof PhotoModel) { + result = spyService.createPhoto((PhotoModel) item, jobId); + } else if (item instanceof VideoModel) { + result = spyService.createVideo((VideoModel) item, jobId); + } + + verify(spyService).sendPostRequest(anyString(), requestBodyCaptor.capture(), any()); + assertEquals(result.get("item_id"), itemId); + assertEquals(result.get("success"), null); + + RequestBody capturedBody = requestBodyCaptor.getValue(); + MultipartBody multipartBody = (MultipartBody) capturedBody; + + Map multipartFormAnswer = + Map.of( + "job_id", jobId.toString(), + "service", exportingService, + "item_id", itemId, + "title", itemName, + "description", description); + + for (MultipartBody.Part part : multipartBody.parts()) { + String partName = + part.headers().get("Content-Disposition").split(";")[1].split("=")[1].replace("\"", ""); + Buffer buffer = new Buffer(); + part.body().writeTo(buffer); + if (partName.equals("file")) { + byte[] partBytes = buffer.readByteArray(); + + assertArrayEquals(mockImage, partBytes); + String fileName = item.getName(); + String contentDisposition = part.headers().get("Content-Disposition"); + assertTrue(contentDisposition.contains("filename=\"" + fileName + "\"")); + } else if (multipartFormAnswer.containsKey(partName)) { + String partValue = buffer.readUtf8(); + assertEquals(multipartFormAnswer.get(partName), partValue); + } + } + } + + @ParameterizedTest( + name = "shouldSendPostRequestWithCorrectFormBodyWithoutDescription [{index}] {0}") + @MethodSource("provideMediaObjectsWithoutDescriptionInTempStore") + public void shouldSendPostRequestWithCorrectFormBodyWithoutDescription(DownloadableFile item) + throws IOException { + byte[] mockImage = new byte[] {1, 2, 3}; + InputStream mockInputStream = new ByteArrayInputStream(mockImage); + InputStreamWrapper streamWrapper = mock(InputStreamWrapper.class); + SynologyDTPService spyService = Mockito.spy(dtpService); + Map dataMap = Map.of("item_id", itemId); + + when(jobStore.getStream(jobId, fetchUrl)).thenReturn(streamWrapper); + when(streamWrapper.getStream()).thenReturn(mockInputStream); + doReturn(Map.of("success", true, "data", dataMap)) + .when(spyService) + .sendPostRequest(anyString(), any(), any()); + + Map result = new HashMap(); + if (item instanceof PhotoModel) { + result = spyService.createPhoto((PhotoModel) item, jobId); + } else if (item instanceof VideoModel) { + result = spyService.createVideo((VideoModel) item, jobId); + } + + verify(spyService).sendPostRequest(anyString(), requestBodyCaptor.capture(), any()); + assertEquals(result.get("item_id"), itemId); + assertEquals(result.get("success"), null); + + RequestBody capturedBody = requestBodyCaptor.getValue(); + MultipartBody multipartBody = (MultipartBody) capturedBody; + + Map multipartFormAnswer = + Map.of( + "job_id", jobId.toString(), + "service", exportingService, + "item_id", itemId, + "title", itemName); + + for (MultipartBody.Part part : multipartBody.parts()) { + String partName = + part.headers().get("Content-Disposition").split(";")[1].split("=")[1].replace("\"", ""); + Buffer buffer = new Buffer(); + part.body().writeTo(buffer); + assertNotEquals("description", partName); + if (partName.equals("file")) { + byte[] partBytes = buffer.readByteArray(); + + assertArrayEquals(mockImage, partBytes); + String fileName = item.getName(); + String contentDisposition = part.headers().get("Content-Disposition"); + assertTrue(contentDisposition.contains("filename=\"" + fileName + "\"")); + } else if (multipartFormAnswer.containsKey(partName)) { + String partValue = buffer.readUtf8(); + assertEquals(multipartFormAnswer.get(partName), partValue); + } + } + } + + @ParameterizedTest(name = "shouldThrowExceptionIfSendPostRequestFailed [{index}] {0}") + @MethodSource("provideMediaObjectsInTempStore") + public void shouldThrowExceptionIfSendPostRequestFailed(DownloadableFile item) + throws IOException { + byte[] mockImage = new byte[] {1, 2, 3}; + InputStream mockInputStream = new ByteArrayInputStream(mockImage); + InputStreamWrapper streamWrapper = mock(InputStreamWrapper.class); + SynologyDTPService spyService = Mockito.spy(dtpService); + + when(jobStore.getStream(jobId, fetchUrl)).thenReturn(streamWrapper); + when(streamWrapper.getStream()).thenReturn(mockInputStream); + doThrow(new SynologyMaxRetriesExceededException("MockException", 3, new Exception("Failed"))) + .when(spyService) + .sendPostRequest(anyString(), any(), any()); + + if (item instanceof PhotoModel) { + assertThrows( + SynologyMaxRetriesExceededException.class, + () -> spyService.createPhoto((PhotoModel) item, jobId), + "MockException"); + } else if (item instanceof VideoModel) { + assertThrows( + SynologyMaxRetriesExceededException.class, + () -> spyService.createVideo((VideoModel) item, jobId), + "MockException"); + } + } + } + + @Nested + public class SendPostRequestTest { + private RequestBody requestBody = RequestBody.create(MediaType.parse("application/json"), "{}"); + + @Test + public void shouldRetryIfGotError() throws IOException { + Call mockCall = mock(Call.class); + + Response mockResponseFail = mock(Response.class); + when(mockResponseFail.code()).thenReturn(SC_INTERNAL_SERVER_ERROR); + when(mockResponseFail.isSuccessful()).thenReturn(false); + + Response mockResponseSuccess = mock(Response.class); + ResponseBody mockResponseSuccessBody = mock(ResponseBody.class); + when(mockResponseSuccess.isSuccessful()).thenReturn(true); + when(mockResponseSuccess.body()).thenReturn(mockResponseSuccessBody); + when(mockResponseSuccessBody.string()).thenReturn("{\"success\": true}"); + + when(client.newCall(any(Request.class))).thenReturn(mockCall); + when(mockCall.execute()).thenReturn(mockResponseFail).thenReturn(mockResponseSuccess); + + Map result = + dtpService.sendPostRequest(TestConfigs.TEST_C2_BASE_URL, requestBody, jobId); + + assertEquals(Map.of("success", true), result); + + verify(tokenManager, never()) + .refreshToken(any(UUID.class), eq(client), any(ObjectMapper.class)); + verify(client, times(2)).newCall(any(Request.class)); + } + + @Test + public void shouldThrowExceptionIfReachMaxRetries() throws IOException { + Call mockCall = mock(Call.class); + + Response mockResponseFail = mock(Response.class); + when(mockResponseFail.code()).thenReturn(SC_INTERNAL_SERVER_ERROR); + when(mockResponseFail.isSuccessful()).thenReturn(false); + + when(client.newCall(any(Request.class))).thenReturn(mockCall); + when(mockCall.execute()).thenReturn(mockResponseFail); + + assertThrows( + SynologyMaxRetriesExceededException.class, + () -> dtpService.sendPostRequest(TestConfigs.TEST_C2_BASE_URL, requestBody, jobId), + String.format("Failed to send POST request %d times", TestConfigs.TEST_MAX_ATTEMPTS)); + verify(tokenManager, never()) + .refreshToken(any(UUID.class), eq(client), any(ObjectMapper.class)); + verify(client, times(TestConfigs.TEST_MAX_ATTEMPTS)).newCall(any(Request.class)); + } + + @Test + public void shouldRefreshTokenAndRetryIfGotUnauthorizedHttpError() + throws IOException, JsonProcessingException { + Call mockCall = mock(Call.class); + + Response mockResponseFail = mock(Response.class); + when(mockResponseFail.code()).thenReturn(SC_UNAUTHORIZED); + when(mockResponseFail.isSuccessful()).thenReturn(false); + + Response mockResponseSuccess = mock(Response.class); + ResponseBody mockResponseSuccessBody = mock(ResponseBody.class); + when(mockResponseSuccess.isSuccessful()).thenReturn(true); + when(mockResponseSuccess.body()).thenReturn(mockResponseSuccessBody); + when(mockResponseSuccessBody.string()).thenReturn("{\"success\": true}"); + + when(client.newCall(any(Request.class))).thenReturn(mockCall); + when(mockCall.execute()).thenReturn(mockResponseFail).thenReturn(mockResponseSuccess); + when(tokenManager.refreshToken(any(UUID.class), eq(client), any(ObjectMapper.class))) + .thenReturn(true); + + Map result = + dtpService.sendPostRequest(TestConfigs.TEST_C2_BASE_URL, requestBody, jobId); + + assertEquals(Map.of("success", true), result); + + verify(tokenManager, times(1)) + .refreshToken(any(UUID.class), eq(client), any(ObjectMapper.class)); + verify(client, times(2)).newCall(any(Request.class)); + } + + @Test + public void shouldInvokeRefreshTokenOnlyOnceIfGotUnauthorizedMultipleTimes() + throws IOException { + Call mockCall = mock(Call.class); + + Response mockResponseFail = mock(Response.class); + when(mockResponseFail.code()).thenReturn(SC_UNAUTHORIZED); + when(mockResponseFail.isSuccessful()).thenReturn(false); + + when(client.newCall(any(Request.class))).thenReturn(mockCall); + when(mockCall.execute()).thenReturn(mockResponseFail); + when(tokenManager.refreshToken(any(UUID.class), eq(client), any(ObjectMapper.class))) + .thenReturn(false); + + assertThrows( + SynologyMaxRetriesExceededException.class, + () -> dtpService.sendPostRequest(TestConfigs.TEST_C2_BASE_URL, requestBody, jobId), + String.format("Failed to send POST request %d times", TestConfigs.TEST_MAX_ATTEMPTS)); + + verify(tokenManager, times(1)) + .refreshToken(any(UUID.class), eq(client), any(ObjectMapper.class)); + verify(client, times(TestConfigs.TEST_MAX_ATTEMPTS)).newCall(any(Request.class)); + } + + @Test + public void shouldThrowExceptionIfParseResponseFailed() throws IOException { + Response mockResponse = mock(Response.class); + ResponseBody mockResponseBody = mock(ResponseBody.class); + Call mockCall = mock(Call.class); + + when(client.newCall(any(Request.class))).thenReturn(mockCall); + when(mockCall.execute()).thenReturn(mockResponse); + when(mockResponse.isSuccessful()).thenReturn(true); + when(mockResponse.body()).thenReturn(mockResponseBody); + when(mockResponseBody.string()) + .thenThrow(new IOException("Error when call response.body.string()")); + + assertThrows( + SynologyMaxRetriesExceededException.class, + () -> dtpService.sendPostRequest(TestConfigs.TEST_C2_BASE_URL, requestBody, jobId), + String.format("Failed to send POST request %d times", TestConfigs.TEST_MAX_ATTEMPTS)); + verify(tokenManager, never()) + .refreshToken(any(UUID.class), eq(client), any(ObjectMapper.class)); + verify(client, times(TestConfigs.TEST_MAX_ATTEMPTS)).newCall(any(Request.class)); + } + + @Test + public void shouldReturnResponseData() throws IOException, JsonProcessingException { + Response mockResponse = mock(Response.class); + ResponseBody mockResponseBody = mock(ResponseBody.class); + Call mockCall = mock(Call.class); + + when(client.newCall(any(Request.class))).thenReturn(mockCall); + when(mockCall.execute()).thenReturn(mockResponse); + when(mockResponse.isSuccessful()).thenReturn(true); + when(mockResponse.body()).thenReturn(mockResponseBody); + when(mockResponseBody.string()).thenReturn("{\"success\": true}"); + + Map result = + dtpService.sendPostRequest(TestConfigs.TEST_C2_BASE_URL, requestBody, jobId); + assertEquals(Map.of("success", true), result); + verify(tokenManager, never()) + .refreshToken(any(UUID.class), eq(client), any(ObjectMapper.class)); + verify(client, times(1)).newCall(any(Request.class)); + } + } +} diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/test/java/org/datatransferproject/datatransfer/synology/service/SynologyOAuthTokenManagerTest.java b/extensions/data-transfer/portability-data-transfer-synology/src/test/java/org/datatransferproject/datatransfer/synology/service/SynologyOAuthTokenManagerTest.java new file mode 100644 index 000000000..8538c053c --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/test/java/org/datatransferproject/datatransfer/synology/service/SynologyOAuthTokenManagerTest.java @@ -0,0 +1,233 @@ +/* + * Copyright 2025 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.datatransferproject.datatransfer.synology.service; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.*; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.util.Map; +import java.util.UUID; +import okhttp3.Call; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; +import org.datatransferproject.api.launcher.Monitor; +import org.datatransferproject.datatransfer.synology.exceptions.SynologyImportException; +import org.datatransferproject.types.transfer.auth.AppCredentials; +import org.datatransferproject.types.transfer.auth.TokensAndUrlAuthData; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class SynologyOAuthTokenManagerTest { + @InjectMocks protected SynologyOAuthTokenManager tokenManager; + @Mock protected Monitor monitor; + @Mock protected AppCredentials appCredentials; + + @BeforeEach + public void setUp() throws Exception { + lenient().when(appCredentials.getKey()).thenReturn("mockClientId"); + lenient().when(appCredentials.getSecret()).thenReturn("mockClientSecret"); + } + + @Nested + class OAuthTokenManagerBasicTest { + @Test + void shouldGetAccessTokenIfExist() { + UUID jobId = UUID.randomUUID(); + TokensAndUrlAuthData authData = + new TokensAndUrlAuthData("accessToken", "refreshToken", "http://mock.token.url"); + tokenManager.addAuthDataIfNotExist(jobId, authData); + + assertEquals("accessToken", tokenManager.getAccessToken(jobId)); + } + + @Test + void shouldThrowExceptionIfJobIdNotFound() { + UUID jobId = UUID.randomUUID(); + + assertThrows( + SynologyImportException.class, + () -> tokenManager.getAccessToken(jobId), + "No auth data found for job: " + jobId); + } + + @Test + void shouldNotReplaceExistingAuthData() { + UUID jobId = UUID.randomUUID(); + TokensAndUrlAuthData authData1 = + new TokensAndUrlAuthData("accessToken1", "refreshToken1", "url1"); + TokensAndUrlAuthData authData2 = + new TokensAndUrlAuthData("accessToken2", "refreshToken2", "url2"); + + tokenManager.addAuthDataIfNotExist(jobId, authData1); + tokenManager.addAuthDataIfNotExist(jobId, authData2); + + String accessToken = tokenManager.getAccessToken(jobId); + assertEquals("accessToken1", accessToken); + } + } + + @Nested + class RefreshTokenTest { + @Test + void shouldReturnFalseIfJobIdNotFound() { + UUID jobId = UUID.randomUUID(); + OkHttpClient mockClient = mock(OkHttpClient.class); + ObjectMapper mockObjectMapper = mock(ObjectMapper.class); + + boolean result = tokenManager.refreshToken(jobId, mockClient, mockObjectMapper); + + assertFalse(result); + } + + @Test + void shouldReturnFalseIfGotIOExceptionWhenSendingRequest() throws IOException { + UUID jobId = UUID.randomUUID(); + TokensAndUrlAuthData authData = + new TokensAndUrlAuthData("oldAccessToken", "oldRefreshToken", "http://mock.token.url"); + tokenManager.addAuthDataIfNotExist(jobId, authData); + + OkHttpClient mockClient = mock(OkHttpClient.class); + ObjectMapper mockObjectMapper = mock(ObjectMapper.class); + Response mockResponse = mock(Response.class); + ResponseBody mockResponseBody = mock(ResponseBody.class); + Call mockCall = mock(Call.class); + + when(mockClient.newCall(any(Request.class))).thenReturn(mockCall); + doThrow(new IOException("Error when call client.execute")).when(mockCall).execute(); + + boolean result = tokenManager.refreshToken(jobId, mockClient, mockObjectMapper); + + assertFalse(result); + } + + @Test + void shouldReturnFalseIfRefreshTokenRequestFailed() throws IOException { + UUID jobId = UUID.randomUUID(); + TokensAndUrlAuthData authData = + new TokensAndUrlAuthData("oldAccessToken", "oldRefreshToken", "http://mock.token.url"); + tokenManager.addAuthDataIfNotExist(jobId, authData); + + OkHttpClient mockClient = mock(OkHttpClient.class); + ObjectMapper mockObjectMapper = mock(ObjectMapper.class); + Response mockResponse = mock(Response.class); + ResponseBody mockResponseBody = mock(ResponseBody.class); + Call mockCall = mock(Call.class); + + when(mockClient.newCall(any(Request.class))).thenReturn(mockCall); + doReturn(mockResponse).when(mockCall).execute(); + when(mockResponse.body()).thenReturn(mockResponseBody); + when(mockResponseBody.string()).thenReturn("{}"); + when(mockResponse.isSuccessful()).thenReturn(false); + + boolean result = tokenManager.refreshToken(jobId, mockClient, mockObjectMapper); + + assertFalse(result); + } + + @Test + void shouldReturnFalseIfParseResponseFailed() throws IOException { + UUID jobId = UUID.randomUUID(); + TokensAndUrlAuthData authData = + new TokensAndUrlAuthData("oldAccessToken", "oldRefreshToken", "http://mock.token.url"); + tokenManager.addAuthDataIfNotExist(jobId, authData); + + OkHttpClient mockClient = mock(OkHttpClient.class); + ObjectMapper mockObjectMapper = mock(ObjectMapper.class); + Response mockResponse = mock(Response.class); + ResponseBody mockResponseBody = mock(ResponseBody.class); + Call mockCall = mock(Call.class); + + when(mockClient.newCall(any(Request.class))).thenReturn(mockCall); + doReturn(mockResponse).when(mockCall).execute(); + when(mockResponse.body()).thenReturn(mockResponseBody); + when(mockResponseBody.string()) + .thenThrow(new IOException("Error when call response.body.string()")); + + boolean result = tokenManager.refreshToken(jobId, mockClient, mockObjectMapper); + + assertFalse(result); + } + + @Test + void shouldReturnFalseIfParseJsonFailed() throws IOException, JsonProcessingException { + UUID jobId = UUID.randomUUID(); + TokensAndUrlAuthData authData = + new TokensAndUrlAuthData("oldAccessToken", "oldRefreshToken", "http://mock.token.url"); + tokenManager.addAuthDataIfNotExist(jobId, authData); + + OkHttpClient mockClient = mock(OkHttpClient.class); + ObjectMapper mockObjectMapper = mock(ObjectMapper.class); + Response mockResponse = mock(Response.class); + ResponseBody mockResponseBody = mock(ResponseBody.class); + Call mockCall = mock(Call.class); + + when(mockClient.newCall(any(Request.class))).thenReturn(mockCall); + doReturn(mockResponse).when(mockCall).execute(); + when(mockResponse.isSuccessful()).thenReturn(true); + when(mockResponse.body()).thenReturn(mockResponseBody); + when(mockResponseBody.string()).thenReturn("{}"); + when(mockObjectMapper.readValue(anyString(), eq(Map.class))) + .thenThrow(new JsonProcessingException("Error when call objectMapper.readValue") {}); + + boolean result = tokenManager.refreshToken(jobId, mockClient, mockObjectMapper); + + assertFalse(result); + } + + @Test + void shouldUpdateTokenAndReturnTrue() throws IOException, JsonProcessingException { + UUID jobId = UUID.randomUUID(); + TokensAndUrlAuthData authData = + new TokensAndUrlAuthData("oldAccessToken", "oldRefreshToken", "http://mock.token.url"); + tokenManager.addAuthDataIfNotExist(jobId, authData); + + OkHttpClient mockClient = mock(OkHttpClient.class); + ObjectMapper mockObjectMapper = mock(ObjectMapper.class); + Response mockResponse = mock(Response.class); + ResponseBody mockResponseBody = mock(ResponseBody.class); + Call mockCall = mock(Call.class); + + when(mockClient.newCall(any(Request.class))).thenReturn(mockCall); + doReturn(mockResponse).when(mockCall).execute(); + when(mockResponse.isSuccessful()).thenReturn(true); + when(mockResponse.body()).thenReturn(mockResponseBody); + when(mockResponseBody.string()).thenReturn("{}"); + when(mockObjectMapper.readValue(anyString(), eq(Map.class))) + .thenReturn(Map.of("access_token", "newAccessToken", "refresh_token", "newRefreshToken")); + + boolean result = tokenManager.refreshToken(jobId, mockClient, mockObjectMapper); + + assertTrue(result); + assertEquals("newAccessToken", tokenManager.getAccessToken(jobId)); + } + } +} diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/test/java/org/datatransferproject/datatransfer/synology/uploader/SynologyUploaderTest.java b/extensions/data-transfer/portability-data-transfer-synology/src/test/java/org/datatransferproject/datatransfer/synology/uploader/SynologyUploaderTest.java new file mode 100644 index 000000000..391f86af2 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/test/java/org/datatransferproject/datatransfer/synology/uploader/SynologyUploaderTest.java @@ -0,0 +1,402 @@ +/* + * Copyright 2025 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.datatransferproject.datatransfer.synology.uploader; + +import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; +import org.datatransferproject.api.launcher.Monitor; +import org.datatransferproject.datatransfer.synology.constant.SynologyConstant; +import org.datatransferproject.datatransfer.synology.exceptions.SynologyHttpException; +import org.datatransferproject.datatransfer.synology.exceptions.SynologyImportException; +import org.datatransferproject.datatransfer.synology.service.SynologyDTPService; +import org.datatransferproject.spi.transfer.idempotentexecutor.IdempotentImportExecutor; +import org.datatransferproject.types.common.DownloadableItem; +import org.datatransferproject.types.common.models.media.MediaAlbum; +import org.datatransferproject.types.common.models.photos.PhotoModel; +import org.datatransferproject.types.common.models.videos.VideoModel; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.TestInstance.Lifecycle; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class SynologyUploaderTest { + @Mock private SynologyDTPService synologyDTPService; + @Mock private IdempotentImportExecutor executor; + @Mock private Monitor monitor; + + UUID mockJobId = UUID.randomUUID(); + + @BeforeEach + public void setUp() throws Exception { + doAnswer( + invocation -> { + Callable lambda = invocation.getArgument(2); + return lambda.call(); + }) + .when(executor) + .executeAndSwallowIOExceptions(anyString(), anyString(), any()); + } + + private boolean containsMessage(Throwable throwable, String expectedMessage) { + while (throwable != null) { + if (throwable.getMessage() != null && throwable.getMessage().contains(expectedMessage)) { + return true; + } + throwable = throwable.getCause(); + } + return false; + } + + @Nested + public class ImportAlbums { + @Test + public void shouldImportAlbums() throws Exception { + SynologyUploader uploader = new SynologyUploader(executor, monitor, synologyDTPService); + + List albums = + Arrays.asList( + new MediaAlbum("1", "album1", "desc"), new MediaAlbum("2", "album2", "desc")); + + for (MediaAlbum album : albums) { + when(synologyDTPService.createAlbum(album, mockJobId)) + .thenReturn(Map.of("album_id", album.getId())); + } + + uploader.importAlbums(albums, mockJobId); + + verify(synologyDTPService, times(albums.size())).createAlbum(any(), any()); + + // should create with cache + for (MediaAlbum album : albums) { + verify(synologyDTPService).createAlbum(album, mockJobId); + verify(executor) + .executeAndSwallowIOExceptions(eq(album.getId()), eq(album.getName()), any()); + } + } + + @Test + public void shouldThrowExceptionIfCreateAlbumFails() { + SynologyUploader spyUploader = + Mockito.spy(new SynologyUploader(executor, monitor, synologyDTPService)); + List albums = List.of(new MediaAlbum("1", "album1", "desc")); + String expectedMessage = "Failed to import albums"; + when(synologyDTPService.createAlbum(any(), any())) + .thenThrow( + new SynologyHttpException( + expectedMessage, SC_INTERNAL_SERVER_ERROR, "Internal Server Error")); + Exception e = + assertThrows( + SynologyImportException.class, () -> spyUploader.importAlbums(albums, mockJobId)); + assertTrue(containsMessage(e, expectedMessage)); + } + } + + @Nested + @TestInstance(Lifecycle.PER_CLASS) + public class ImportMedia { + private final List dataIds = List.of("dataId1", "dataId2"); + private final List newItemIds = List.of("1", "2"); + private final String albumId = "1"; + private final String newAlbumId = "10"; + private final Map albumIdToNewAlbumIdMap = Map.of(albumId, newAlbumId); + private final Map dataIdToItemIdMap = + IntStream.range(0, dataIds.size()) + .boxed() + .collect(Collectors.toMap(dataIds::get, newItemIds::get)); + private final List photos = + List.of( + new PhotoModel("title1", "url1", "desc", "mediaType", dataIds.get(0), albumId, true), + new PhotoModel("title2", "url2", "desc", "mediaType", dataIds.get(1), albumId, true)); + private final List videos = + List.of( + new VideoModel("name1", "url1", "desc", "format", dataIds.get(0), albumId, true, null), + new VideoModel("name2", "url2", "desc", "format", dataIds.get(1), albumId, true, null)); + + private Stream provideMediaItems() { + return Stream.of(photos, videos); + } + + @BeforeEach + public void setUp() { + lenient() + .when(synologyDTPService.addItemToAlbum(any(), any(), any())) + .thenReturn(Map.of("success", true)); + lenient() + .when(synologyDTPService.createAlbum(any(), any())) + .thenAnswer( + invocation -> { + MediaAlbum album = invocation.getArgument(0); + return Map.of("album_id", albumIdToNewAlbumIdMap.get(album.getId())); + }); + for (PhotoModel photo : photos) { + lenient() + .when(synologyDTPService.createPhoto(photo, mockJobId)) + .thenAnswer( + invocation -> { + PhotoModel photoModel = invocation.getArgument(0); + return Map.of("item_id", dataIdToItemIdMap.get(photoModel.getDataId())); + }); + } + for (VideoModel video : videos) { + lenient() + .when(synologyDTPService.createVideo(video, mockJobId)) + .thenAnswer( + invocation -> { + VideoModel videoModel = invocation.getArgument(0); + return Map.of("item_id", dataIdToItemIdMap.get(videoModel.getDataId())); + }); + } + } + + @ParameterizedTest(name = "shouldImportMediaItemsWhenAlbumBeforeItem [{index}] {0}") + @MethodSource("provideMediaItems") + public void shouldImportMediaItemsWhenAlbumBeforeItem( + List mediaItems) { + SynologyUploader uploader = new SynologyUploader(executor, monitor, synologyDTPService); + List albums = List.of(new MediaAlbum(albumId, "album1", "desc")); + + uploader.importAlbums(albums, mockJobId); + if (mediaItems.get(0) instanceof PhotoModel) { + uploader.importPhotos((List) mediaItems, mockJobId); + } else if (mediaItems.get(0) instanceof VideoModel) { + uploader.importVideos((List) mediaItems, mockJobId); + } + + for (int i = 0; i < mediaItems.size(); i++) { + Object media = mediaItems.get(i); + if (media instanceof PhotoModel) { + verify(synologyDTPService).createPhoto((PhotoModel) media, mockJobId); + } else if (media instanceof VideoModel) { + verify(synologyDTPService).createVideo((VideoModel) media, mockJobId); + } + verify(synologyDTPService).addItemToAlbum(newAlbumId, newItemIds.get(i), mockJobId); + } + } + + @ParameterizedTest(name = "shouldImportMediaItemsWhenItemBeforeAlbum [{index}] {0}") + @MethodSource("provideMediaItems") + public void shouldImportMediaItemsWhenItemBeforeAlbum( + List mediaItems) throws Exception { + SynologyUploader uploader = new SynologyUploader(executor, monitor, synologyDTPService); + List albums = List.of(new MediaAlbum(albumId, "album1", "desc")); + + if (mediaItems.get(0) instanceof PhotoModel) { + uploader.importPhotos((List) mediaItems, mockJobId); + } else if (mediaItems.get(0) instanceof VideoModel) { + uploader.importVideos((List) mediaItems, mockJobId); + } + uploader.importAlbums(albums, mockJobId); + + for (int i = 0; i < mediaItems.size(); i++) { + Object media = mediaItems.get(i); + if (media instanceof PhotoModel) { + verify(synologyDTPService).createPhoto((PhotoModel) media, mockJobId); + } else if (media instanceof VideoModel) { + verify(synologyDTPService).createVideo((VideoModel) media, mockJobId); + } + verify(synologyDTPService).addItemToAlbum(newAlbumId, newItemIds.get(i), mockJobId); + } + } + + @ParameterizedTest(name = "shouldImportMediaItemsWithCache [{index}] {0}") + @MethodSource("provideMediaItems") + public void shouldImportMediaItemsWithCache(List mediaItems) + throws Exception { + SynologyUploader uploader = new SynologyUploader(executor, monitor, synologyDTPService); + List albums = List.of(new MediaAlbum(albumId, "album1", "desc")); + + uploader.importAlbums(albums, mockJobId); + if (mediaItems.get(0) instanceof PhotoModel) { + uploader.importPhotos((List) mediaItems, mockJobId); + } else if (mediaItems.get(0) instanceof VideoModel) { + uploader.importVideos((List) mediaItems, mockJobId); + } + + for (Object media : mediaItems) { + if (media instanceof PhotoModel) { + verify(executor) + .executeAndSwallowIOExceptions( + eq(((PhotoModel) media).getDataId()), eq(((PhotoModel) media).getTitle()), any()); + } else if (media instanceof VideoModel) { + verify(executor) + .executeAndSwallowIOExceptions( + eq(((VideoModel) media).getDataId()), eq(((VideoModel) media).getName()), any()); + } + String newItemId = newItemIds.get(mediaItems.indexOf(media)); + String albumIdToItemId = + String.format(SynologyConstant.ALBUM_ITEM_ID_FORMAT, newAlbumId, newItemId); + verify(executor).executeAndSwallowIOExceptions(eq(albumIdToItemId), eq(newItemId), any()); + } + } + + @ParameterizedTest( + name = "shouldThrowHttpExceptionWithStatusCodeIfCreateMediaItemFails [{index}] {0}") + @MethodSource("provideMediaItems") + public void shouldThrowHttpExceptionWithStatusCodeIfCreateMediaItemFails( + List mediaItems) { + SynologyUploader spyUploader = + Mockito.spy(new SynologyUploader(executor, monitor, synologyDTPService)); + List albums = List.of(new MediaAlbum("1", "album1", "desc")); + String expectedMessage = String.format("statusCode=%d", SC_INTERNAL_SERVER_ERROR); + + if (mediaItems.get(0) instanceof PhotoModel) { + when(synologyDTPService.createPhoto(any(), any())) + .thenThrow( + new SynologyHttpException( + "Failed to create photo", SC_INTERNAL_SERVER_ERROR, "Internal Server Error")); + } else if (mediaItems.get(0) instanceof VideoModel) { + when(synologyDTPService.createVideo(any(), any())) + .thenThrow( + new SynologyHttpException( + "Failed to create video", SC_INTERNAL_SERVER_ERROR, "Internal Server Error")); + } + + spyUploader.importAlbums(albums, mockJobId); + + if (mediaItems.get(0) instanceof PhotoModel) { + Exception e = + assertThrows( + SynologyImportException.class, + () -> spyUploader.importPhotos((List) mediaItems, mockJobId)); + assertTrue(containsMessage(e, expectedMessage)); + } else if (mediaItems.get(0) instanceof VideoModel) { + Exception e = + assertThrows( + SynologyImportException.class, + () -> spyUploader.importVideos((List) mediaItems, mockJobId)); + assertTrue(containsMessage(e, expectedMessage)); + } + } + + @ParameterizedTest(name = "shouldThrowExceptionIfCreateMediaItemNotSuccess [{index}] {0}") + @MethodSource("provideMediaItems") + public void shouldThrowExceptionIfCreateMediaItemNotSuccess( + List mediaItems) { + SynologyUploader spyUploader = + Mockito.spy(new SynologyUploader(executor, monitor, synologyDTPService)); + List albums = List.of(new MediaAlbum("1", "album1", "desc")); + + if (mediaItems.get(0) instanceof PhotoModel) { + when(synologyDTPService.createPhoto(any(), any())).thenReturn(Map.of("success", false)); + } else if (mediaItems.get(0) instanceof VideoModel) { + when(synologyDTPService.createVideo(any(), any())).thenReturn(Map.of("success", false)); + } + + spyUploader.importAlbums(albums, mockJobId); + + if (mediaItems.get(0) instanceof PhotoModel) { + Exception e = + assertThrows( + SynologyImportException.class, + () -> spyUploader.importPhotos((List) mediaItems, mockJobId)); + assertTrue(containsMessage(e, "Failed to import photos")); + } else if (mediaItems.get(0) instanceof VideoModel) { + Exception e = + assertThrows( + SynologyImportException.class, + () -> spyUploader.importVideos((List) mediaItems, mockJobId)); + assertTrue(containsMessage(e, "Failed to import videos")); + } + } + + @ParameterizedTest(name = "shouldThrowExceptionIfAddItemToAlbumFails [{index}] {0}") + @MethodSource("provideMediaItems") + public void shouldThrowExceptionIfAddItemToAlbumFails( + List mediaItems) { + SynologyUploader spyUploader = + Mockito.spy(new SynologyUploader(executor, monitor, synologyDTPService)); + List albums = List.of(new MediaAlbum("1", "album1", "desc")); + String expectedMessage = "Failed to add item to album"; + + when(synologyDTPService.addItemToAlbum(any(), any(), any())) + .thenThrow( + new SynologyHttpException( + expectedMessage, SC_INTERNAL_SERVER_ERROR, "Internal Server Error")); + + spyUploader.importAlbums(albums, mockJobId); + if (mediaItems.get(0) instanceof PhotoModel) { + Exception e = + assertThrows( + SynologyImportException.class, + () -> spyUploader.importPhotos((List) mediaItems, mockJobId)); + assertTrue(containsMessage(e, expectedMessage)); + } else if (mediaItems.get(0) instanceof VideoModel) { + Exception e = + assertThrows( + SynologyImportException.class, + () -> spyUploader.importVideos((List) mediaItems, mockJobId), + expectedMessage); + assertTrue(containsMessage(e, expectedMessage)); + } + } + + @ParameterizedTest(name = "shouldThrowExceptionIfAddItemToAlbumNotSuccess [{index}] {0}") + @MethodSource("provideMediaItems") + public void shouldThrowExceptionIfAddItemToAlbumNotSuccess( + List mediaItems) { + SynologyUploader spyUploader = + Mockito.spy(new SynologyUploader(executor, monitor, synologyDTPService)); + List albums = List.of(new MediaAlbum("1", "album1", "desc")); + String expectedMessage = "Unsuccessful"; + + when(synologyDTPService.addItemToAlbum(any(), any(), any())) + .thenReturn(Map.of("success", false)); + + spyUploader.importAlbums(albums, mockJobId); + if (mediaItems.get(0) instanceof PhotoModel) { + Exception e = + assertThrows( + SynologyImportException.class, + () -> spyUploader.importPhotos((List) mediaItems, mockJobId)); + assertTrue(containsMessage(e, expectedMessage)); + } else if (mediaItems.get(0) instanceof VideoModel) { + Exception e = + assertThrows( + SynologyImportException.class, + () -> spyUploader.importVideos((List) mediaItems, mockJobId)); + assertTrue(containsMessage(e, expectedMessage)); + } + } + } +} diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/test/java/org/datatransferproject/datatransfer/synology/utils/SynologyMediaAlbumBinderConcurrencyTest.java b/extensions/data-transfer/portability-data-transfer-synology/src/test/java/org/datatransferproject/datatransfer/synology/utils/SynologyMediaAlbumBinderConcurrencyTest.java new file mode 100644 index 000000000..c2362e410 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/test/java/org/datatransferproject/datatransfer/synology/utils/SynologyMediaAlbumBinderConcurrencyTest.java @@ -0,0 +1,104 @@ +/* + * Copyright 2025 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.datatransferproject.datatransfer.synology.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.UUID; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.BiFunction; +import org.datatransferproject.api.launcher.Monitor; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class SynologyMediaAlbumBinderConcurrencyTest { + @Mock private Monitor monitor; + + @Test + public void shouldProcessAllItems() throws InterruptedException { + int latch_timeout_sec = 5; + int numPutJobs = 10; + int numWhenAlbumReadyJobs = 10; + String albumId = "1"; + String newAlbumId = "10"; + + ConcurrentLinkedQueue processedItems = new ConcurrentLinkedQueue<>(); + OnAlbumReadyCallback addToProcessedItems = + (__, itemKey, ___) -> { + processedItems.add(itemKey); + }; + SynologyMediaAlbumBinder binder = + new SynologyMediaAlbumBinder<>(addToProcessedItems, monitor); + + ExecutorService executor = Executors.newFixedThreadPool(10); + CountDownLatch startLatch = new CountDownLatch(1); + CountDownLatch doneLatch = new CountDownLatch(numPutJobs + numWhenAlbumReadyJobs); + + BiFunction getItemId = + (offset, itemId) -> String.valueOf(Integer.parseInt(offset) + itemId); + + UUID jobId = UUID.randomUUID(); + + for (int i = 0; i < numPutJobs; i++) { + final String itemId = getItemId.apply(albumId, i); + executor.submit( + () -> { + try { + startLatch.await(); + binder.put(albumId, itemId, jobId); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + doneLatch.countDown(); + } + }); + } + + for (int i = 0; i < numWhenAlbumReadyJobs; i++) { + executor.submit( + () -> { + try { + startLatch.await(); + binder.whenAlbumReady(albumId, newAlbumId, jobId); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } finally { + doneLatch.countDown(); + } + }); + } + + startLatch.countDown(); + assertTrue(doneLatch.await(latch_timeout_sec, TimeUnit.SECONDS)); + executor.shutdown(); + + assertEquals(numPutJobs, processedItems.size()); + for (int i = 0; i < 10; i++) { + String itemId = getItemId.apply(albumId, i); + assertTrue(processedItems.contains(itemId), "Missing item " + itemId); + } + } +} diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/test/java/org/datatransferproject/datatransfer/synology/utils/SynologyMediaAlbumBinderTest.java b/extensions/data-transfer/portability-data-transfer-synology/src/test/java/org/datatransferproject/datatransfer/synology/utils/SynologyMediaAlbumBinderTest.java new file mode 100644 index 000000000..ac41cca98 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/test/java/org/datatransferproject/datatransfer/synology/utils/SynologyMediaAlbumBinderTest.java @@ -0,0 +1,109 @@ +/* + * Copyright 2025 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.datatransferproject.datatransfer.synology.utils; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +import java.util.UUID; +import org.datatransferproject.api.launcher.Monitor; +import org.datatransferproject.types.common.models.media.MediaAlbum; +import org.datatransferproject.types.common.models.photos.PhotoModel; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class SynologyMediaAlbumBinderTest { + @Mock private OnAlbumReadyCallback mockConsumer; + @Mock private Monitor monitor; + private final UUID jobId = UUID.randomUUID(); + private final PhotoModel photo = + new PhotoModel("title1", "url1", "desc", "mediaType", "dataId1", "1", true); + private final MediaAlbum album = new MediaAlbum("1", "album1", "description1"); + private final String newPhotoId = "10"; + private final String newAlbumId = "100"; + + @Test + public void shouldInvokeReadyHandlerWhenCreateItemBeforeAlbum() { + SynologyMediaAlbumBinder binder = new SynologyMediaAlbumBinder<>(mockConsumer, monitor); + + binder.put(photo.getAlbumId(), newPhotoId, jobId); + verify(mockConsumer, never()).accept(any(), any(), any()); + + binder.whenAlbumReady(album.getId(), newAlbumId, jobId); + verify(mockConsumer).accept(eq(newAlbumId), eq(newPhotoId), eq(jobId)); + } + + @Test + public void shouldInvokeReadyHandlerWhenCreateAlbumBeforeItem() { + SynologyMediaAlbumBinder binder = new SynologyMediaAlbumBinder<>(mockConsumer, monitor); + + binder.whenAlbumReady(album.getId(), newAlbumId, jobId); + verify(mockConsumer, never()).accept(any(), any(), any()); + + binder.put(photo.getAlbumId(), newPhotoId, jobId); + verify(mockConsumer).accept(eq(newAlbumId), eq(newPhotoId), eq(jobId)); + } + + @Test + public void shouldNotInvokeReadyHandlerWhenCreateItemBeforeAlbumAndKeyNotMatch() { + SynologyMediaAlbumBinder binder = new SynologyMediaAlbumBinder<>(mockConsumer, monitor); + + binder.put(photo.getAlbumId(), newPhotoId, jobId); + verify(mockConsumer, never()).accept(any(), any(), any()); + + binder.whenAlbumReady("2", newAlbumId, jobId); + verify(mockConsumer, never()).accept(any(), any(), any()); + } + + @Test + public void shouldNotInvokeReadyHandlerWhenCreateAlbumBeforeItemAndKeyNotMatch() { + SynologyMediaAlbumBinder binder = new SynologyMediaAlbumBinder<>(mockConsumer, monitor); + + binder.whenAlbumReady(album.getId(), newAlbumId, jobId); + verify(mockConsumer, never()).accept(any(), any(), any()); + + binder.put("2", newPhotoId, jobId); + verify(mockConsumer, never()).accept(any(), any(), any()); + } + + @Test + public void shouldNotInvokeReadyHandlerWhenAlbumKeyIsNull() { + SynologyMediaAlbumBinder binder = new SynologyMediaAlbumBinder<>(mockConsumer, monitor); + binder.put(null, newPhotoId, jobId); + verify(mockConsumer, never()).accept(any(), any(), any()); + } + + @Test + public void shouldNotInvokeReadyHandlerWhenNewItemKeyIsNull() { + SynologyMediaAlbumBinder binder = new SynologyMediaAlbumBinder<>(mockConsumer, monitor); + binder.put(photo.getAlbumId(), null, jobId); + verify(mockConsumer, never()).accept(any(), any(), any()); + } + + @Test + public void shouldNotInvokeReadyHandlerWhenNewAlbumKeyIsNull() { + SynologyMediaAlbumBinder binder = new SynologyMediaAlbumBinder<>(mockConsumer, monitor); + binder.whenAlbumReady(album.getId(), null, jobId); + verify(mockConsumer, never()).accept(any(), any(), any()); + } +} diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/test/java/org/datatransferproject/datatransfer/synology/utils/TestConfigs.java b/extensions/data-transfer/portability-data-transfer-synology/src/test/java/org/datatransferproject/datatransfer/synology/utils/TestConfigs.java new file mode 100644 index 000000000..87ea1bf05 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/test/java/org/datatransferproject/datatransfer/synology/utils/TestConfigs.java @@ -0,0 +1,54 @@ +/* + * Copyright 2025 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.datatransferproject.datatransfer.synology.utils; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import org.datatransferproject.datatransfer.synology.models.C2Api; +import org.datatransferproject.datatransfer.synology.models.ServiceConfig; + +public class TestConfigs { + public static final String TEST_C2_BASE_URL = "https://fake.url"; + public static final String TEST_CREATE_ALBUM_PATH = "/create"; + public static final String TEST_UPLOAD_ITEM_PATH = "/upload"; + public static final String TEST_ADD_ITEM_TO_ALBUM_PATH = "/add"; + public static final int TEST_MAX_ATTEMPTS = 5; + + public static ServiceConfig createServiceConfig() { + C2Api.ApiPath apiPath = + new C2Api.ApiPath( + TEST_CREATE_ALBUM_PATH, TEST_UPLOAD_ITEM_PATH, TEST_ADD_ITEM_TO_ALBUM_PATH); + + C2Api c2Api = new C2Api(TEST_C2_BASE_URL, apiPath); + + Map serviceMap = new HashMap<>(); + serviceMap.put("c2", c2Api); + + ServiceConfig.Retry retry = new ServiceConfig.Retry(TEST_MAX_ATTEMPTS); + + return new ServiceConfig(retry, serviceMap); + } + + public static Optional createServiceConfigJson() { + ObjectMapper mapper = new ObjectMapper(); + return Optional.of(mapper.valueToTree(createServiceConfig())); + } +} diff --git a/extensions/data-transfer/portability-data-transfer-synology/src/test/java/org/datatransferproject/datatransfer/synology/utils/UrlUtilsTest.java b/extensions/data-transfer/portability-data-transfer-synology/src/test/java/org/datatransferproject/datatransfer/synology/utils/UrlUtilsTest.java new file mode 100644 index 000000000..b932c8b40 --- /dev/null +++ b/extensions/data-transfer/portability-data-transfer-synology/src/test/java/org/datatransferproject/datatransfer/synology/utils/UrlUtilsTest.java @@ -0,0 +1,69 @@ +/* + * Copyright 2025 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.datatransferproject.datatransfer.synology.utils; + +import static org.junit.jupiter.api.Assertions.*; + +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +public class UrlUtilsTest { + @Nested + public class Join { + @Test + public void testWithTrailingSlash() { + assertEquals("http://example.com/api", UrlUtils.join("http://example.com/", "api")); + assertEquals("http://example.com/api", UrlUtils.join("http://example.com/", "/api")); + assertEquals("http://example.com/api/v1", UrlUtils.join("http://example.com/", "api/v1")); + } + + @Test + public void testWithoutTrailingSlash() { + assertEquals("http://example.com/api", UrlUtils.join("http://example.com", "/api")); + assertEquals("http://example.com/api/v1", UrlUtils.join("http://example.com", "api/v1")); + } + + @Test + public void testWithEmptyUrl() { + assertThrows( + IllegalArgumentException.class, + () -> { + UrlUtils.join("", "api"); + }); + } + + @Test + public void testWithEmptyPath() { + assertEquals("http://example.com/", UrlUtils.join("http://example.com", "")); + } + + @Test + public void testWithNullUrlOrPath() { + assertThrows( + NullPointerException.class, + () -> { + UrlUtils.join(null, "api"); + }); + assertThrows( + NullPointerException.class, + () -> { + UrlUtils.join("http://example.com", null); + }); + } + } +} diff --git a/extensions/security/portability-security-jwe/src/main/java/org/datatransferproject/security/jwe/JWEKeyGenerator.java b/extensions/security/portability-security-jwe/src/main/java/org/datatransferproject/security/jwe/JWEKeyGenerator.java index 6cab92441..066461f5e 100644 --- a/extensions/security/portability-security-jwe/src/main/java/org/datatransferproject/security/jwe/JWEKeyGenerator.java +++ b/extensions/security/portability-security-jwe/src/main/java/org/datatransferproject/security/jwe/JWEKeyGenerator.java @@ -42,7 +42,7 @@ public WorkerKeyPair generate() { monitor.severe(() -> "NoSuchAlgorithmException for: " + ALGORITHM, e); throw new RuntimeException("NoSuchAlgorithmException generating key", e); } - kpg.initialize(1024); + kpg.initialize(2048); KeyPair keyPair = kpg.genKeyPair(); monitor.debug(() -> "JWEKeyGenerator generated WorkerKeyPair"); return new WorkerKeyPair() { diff --git a/settings.gradle b/settings.gradle index 249302312..01bc808ff 100644 --- a/settings.gradle +++ b/settings.gradle @@ -67,6 +67,9 @@ include ':extensions:data-transfer:portability-data-transfer-smugmug' // Spotify include ':extensions:auth:portability-auth-spotify' include ':extensions:data-transfer:portability-data-transfer-spotify' +// Synology +include ':extensions:auth:portability-auth-synology' +include ':extensions:data-transfer:portability-data-transfer-synology' // Twitter include ':extensions:auth:portability-auth-twitter' include ':extensions:data-transfer:portability-data-transfer-twitter'