Skip to content

Commit d4e076b

Browse files
freddyDOTCMSfreddyDOTCMS
authored
Isuue 20325 evaluate changing the bundle output to tar gzip in push publisher (#20428)
* #19646 Fixing error in static publish * #20325 Chang PushPublisher to us the new TarGzipBundleOutput * #20325 Testing * removing spaces * #20325 Refactoring the test * Removing import * #20325 Testing
1 parent 6885d0e commit d4e076b

7 files changed

Lines changed: 131 additions & 76 deletions

File tree

‎dotCMS/src/integration-test/java/com/dotcms/datagen/PushPublishingEndPointDataGen.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import com.dotcms.publisher.endpoint.bean.impl.PushPublishingEndPoint;
44
import com.dotcms.publisher.environment.bean.Environment;
55
import com.dotmarketing.business.APILocator;
6+
import com.dotmarketing.cms.factories.PublicEncryptionFactory;
67
import com.dotmarketing.exception.DotDataException;
78
import com.dotmarketing.exception.DotRuntimeException;
89

@@ -11,7 +12,7 @@ public class PushPublishingEndPointDataGen extends AbstractDataGen<PushPublishin
1112

1213
private String address = "127.0.0.1";
1314
private boolean sending = false;
14-
private String authKey = "123";
15+
private String authKey = "123567";
1516
private String port = "8080";
1617
private boolean enabled = true;
1718
private String serverName = null;
@@ -57,14 +58,15 @@ public PushPublishingEndPoint next() {
5758
final PushPublishingEndPoint publishingEndPoint = new PushPublishingEndPoint();
5859
publishingEndPoint.setAddress(address);
5960
publishingEndPoint.setSending(sending);
60-
publishingEndPoint.setAuthKey(authKey);
61+
publishingEndPoint.setAuthKey(PublicEncryptionFactory.encryptString(authKey));
6162
publishingEndPoint.setPort(port);
6263
publishingEndPoint.setEnabled(enabled);
6364
publishingEndPoint.setServerName(
6465
new StringBuilder(serverName == null ? "ServerName_" + System.currentTimeMillis()
6566
: serverName)
6667
);
6768
publishingEndPoint.setGroupId(environment.getId());
69+
publishingEndPoint.setProtocol("http");
6870

6971
return publishingEndPoint;
7072
}

‎dotCMS/src/integration-test/java/com/dotcms/enterprise/publishing/remote/bundler/DependencyBundlerTest.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1091,7 +1091,6 @@ public void excludeContenletChildAssetByModDate(ModDateTestData modDateTestData)
10911091
publishingEndPoint,
10921092
bundle);
10931093

1094-
10951094
bundler.setConfig(config);
10961095
bundler.generate(bundleOutput, status);
10971096

‎dotCMS/src/integration-test/java/com/dotcms/publisher/business/PublisherTest.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import com.dotcms.publishing.DotPublishingException;
2121
import com.dotcms.publishing.FilterDescriptor;
2222
import com.dotcms.publishing.PublishStatus;
23+
import com.dotcms.publishing.PublisherAPIImplTest;
2324
import com.dotcms.publishing.PublisherConfig;
2425
import com.dotcms.publishing.PublisherConfig.Operation;
2526
import com.dotcms.repackage.org.apache.struts.Globals;
@@ -213,12 +214,18 @@ private void assertPushFolderPage(final FolderPage folderPage, final PushResult
213214
assertTrue(bundlerStatus.isPresent());
214215
assertTrue("We should have 1 page on: " + folderPage.folder, bundlerStatus.get().getCount() != 0);
215216

217+
File tarGzipFile = new File(pushResult.bundlePath + ".tar.gz");
218+
assertTrue(tarGzipFile.exists());
219+
220+
PublisherAPIImplTest.extractTarArchive(tarGzipFile, new File(pushResult.bundlePath));
221+
216222
assertTrue(PublisherTestUtil.existsFolder(pushResult.bundlePath, folderPage.folder));
217223
assertTrue(PublisherTestUtil.existsPage(pushResult.bundlePath, host, folderPage.folder, folderPage.page));
218224
}
219225

220226
private void assertRemoveFolder(final FolderPage folderPage, final PushResult pushResult) throws Exception {
221-
227+
File tarGzipFile = new File(pushResult.bundlePath + ".tar.gz");
228+
PublisherAPIImplTest.extractTarArchive(tarGzipFile, new File(pushResult.bundlePath));
222229
assertTrue(PublisherTestUtil.existsFolder(pushResult.bundlePath, folderPage.folder));
223230
}
224231

‎dotCMS/src/integration-test/java/com/dotcms/publisher/business/TestPushPublisher.java‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -45,16 +45,8 @@ public PublisherConfig process (final PublishStatus status ) throws DotPublishin
4545
File bundleRoot = BundlerUtil.getBundleRoot(this.config);
4646
ArrayList<File> list = new ArrayList<File>(1);
4747
list.add(bundleRoot);
48-
File bundle = new File(bundleRoot+File.separator+".."+File.separator+this.config.getId()+".tar.gz");
48+
File bundle = new File(bundleRoot + ".tar.gz");
4949

50-
// If the tar.gz doesn't exist or if it the first try to push bundle
51-
// we need to compress the bundle folder into the tar.gz file.
52-
if (!bundle.exists() || !pubAuditAPI.isPublishRetry(config.getId())) {
53-
PushUtils.compressFiles(list, bundle, bundleRoot.getAbsolutePath());
54-
} else {
55-
Logger.info(this, "Retrying bundle: " + config.getId()
56-
+ ", we don't need to compress bundle again");
57-
}
5850

5951
List<Environment> environments = APILocator.getEnvironmentAPI().findEnvironmentsByBundleId(this.config.getId());
6052

‎dotCMS/src/integration-test/java/com/dotcms/publishing/PublisherAPIImplTest.java‎

Lines changed: 108 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,16 @@
66
import com.dotcms.contenttype.transform.contenttype.StructureTransformer;
77
import com.dotcms.datagen.*;
88
import com.dotcms.languagevariable.business.LanguageVariableAPI;
9+
import com.dotcms.publisher.bundle.bean.Bundle;
10+
import com.dotcms.publisher.bundle.business.BundleFactoryImpl;
11+
import com.dotcms.publisher.business.DotPublisherException;
12+
import com.dotcms.publisher.business.PublishAuditAPI;
13+
import com.dotcms.publisher.business.PublishAuditHistory;
14+
import com.dotcms.publisher.business.PublishAuditStatus;
15+
import com.dotcms.publisher.endpoint.bean.impl.PushPublishingEndPoint;
16+
import com.dotcms.publisher.environment.bean.Environment;
917
import com.dotcms.publisher.pusher.PushPublisher;
1018
import com.dotcms.publisher.pusher.PushPublisherConfig;
11-
import com.dotcms.publisher.receiver.BundlePublisher;
12-
import com.dotcms.publishing.output.BundleOutput;
13-
import com.dotcms.publishing.output.DirectoryBundleOutput;
14-
import com.dotcms.publishing.output.TarGzipBundleOutput;
1519
import com.dotcms.test.util.FileTestUtil;
1620
import com.dotcms.util.IntegrationTestInitService;
1721
import com.dotmarketing.beans.Host;
@@ -40,18 +44,25 @@
4044
import com.liferay.portal.model.User;
4145
import com.liferay.util.FileUtil;
4246
import com.liferay.util.StringPool;
47+
import com.sun.net.httpserver.HttpExchange;
48+
import com.sun.net.httpserver.HttpHandler;
49+
import com.sun.net.httpserver.HttpServer;
4350
import com.tngtech.java.junit.dataprovider.DataProvider;
4451
import com.tngtech.java.junit.dataprovider.DataProviderRunner;
4552
import com.tngtech.java.junit.dataprovider.UseDataProvider;
4653
import java.io.BufferedInputStream;
4754
import java.io.FileInputStream;
55+
import java.io.InputStream;
4856
import java.io.OutputStream;
57+
import java.net.HttpURLConnection;
58+
import java.net.InetSocketAddress;
4959
import java.nio.file.Files;
5060
import java.nio.file.Paths;
5161
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
5262
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
5363
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
54-
import org.elasticsearch.index.fielddata.FieldData;
64+
import org.apache.commons.io.FileUtils;
65+
5566
import org.jetbrains.annotations.Nullable;
5667
import org.junit.AfterClass;
5768
import org.junit.BeforeClass;
@@ -64,20 +75,14 @@
6475
import java.util.stream.Collectors;
6576

6677
import static com.dotcms.util.CollectionsUtils.*;
67-
import static org.jgroups.util.Util.assertEquals;
78+
import static org.junit.Assert.assertEquals;
79+
import static org.junit.Assert.assertTrue;
6880

6981
@RunWith(DataProviderRunner.class)
7082
public class PublisherAPIImplTest {
7183

7284
private static Contentlet languageVariableCreated;
7385

74-
public static class PushPublisherMock extends PushPublisher {
75-
@Override
76-
public PublisherConfig process ( final PublishStatus status ) throws DotPublishingException {
77-
return this.config;
78-
}
79-
}
80-
8186
public static void prepare() throws Exception {
8287

8388
//Setting web app environment
@@ -125,7 +130,7 @@ public static void createLanguageVariableIfNeeded() throws DotSecurityException,
125130
public static Object[] publishers() throws Exception {
126131
prepare();
127132

128-
final List<TestAsset> assets = list(
133+
return new TestAsset[]{
129134
getContentTypeWithHost(),
130135
getTemplateWithDependencies(),
131136
getContainerWithDependencies(),
@@ -136,21 +141,7 @@ public static Object[] publishers() throws Exception {
136141
getLanguageWithDependencies(),
137142
getRuleWithDependencies(),
138143
getContentWithSeveralVersions()
139-
);
140-
final List<Class<? extends Publisher>> publishers = list(
141-
GenerateBundlePublisher.class,
142-
PushPublisherMock.class
143-
);
144-
145-
final List<TestCase> cases = new ArrayList<>();
146-
147-
for (final Class<? extends Publisher> publisher : publishers) {
148-
for (TestAsset asset : assets) {
149-
cases.add(new TestCase(publisher, asset));
150-
}
151-
}
152-
153-
return cases.toArray();
144+
};
154145
}
155146

156147
private static TestAsset getContentWithSeveralVersions() throws DotDataException, DotSecurityException {
@@ -385,20 +376,19 @@ private static TestAsset getContainerWithDependencies() throws DotDataException,
385376
}
386377

387378
/**
388-
* Method to Test: {@link PublisherAPIImpl#publish(PublisherConfig, BundleOutput)}
389-
* When: Add different assets into a bundle
379+
* Method to Test: {@link PublisherAPIImpl#publish(PublisherConfig)}
380+
* When: Add different assets into a bundle, and generate it
390381
* Should: Create all the files
391382
*/
392383
@Test
393384
@UseDataProvider("publishers")
394-
public void publish(final TestCase testCase) throws DotPublishingException, DotSecurityException, IOException, DotDataException {
395-
final Class<? extends Publisher> publisher = testCase.publisher;
396-
final TestAsset testAsset = testCase.asset;
385+
public void generateBundle(final TestAsset testAsset) throws DotPublishingException, DotSecurityException, IOException, DotDataException {
386+
final Class<? extends Publisher> publisher = GenerateBundlePublisher.class;
397387
final Collection<Object> dependencies = new HashSet<>();
398388
dependencies.addAll(testAsset.expectedInBundle);
399389

400390
createLanguageVariableIfNeeded();
401-
addLanguageVariableDependencies(dependencies, testCase.asset.addLanguageVariableDependencies);
391+
addLanguageVariableDependencies(dependencies, testAsset.addLanguageVariableDependencies);
402392

403393
final FilterDescriptor filterDescriptor = new FilterDescriptorDataGen().nextPersisted();
404394

@@ -420,13 +410,92 @@ public void publish(final TestCase testCase) throws DotPublishingException, DotS
420410
final PublishStatus publish = publisherAPI.publish(config);
421411
File bundleRoot = publish.getOutputFiles().get(0);
422412

423-
if (GenerateBundlePublisher.class.equals(publisher)) {
413+
final File extractHere = new File(bundleRoot.getParent() + File.separator + config.getName());
414+
extractTarArchive(bundleRoot, extractHere);
415+
assertBundle(testAsset, dependencies, extractHere);
416+
}
417+
418+
/**
419+
* Method to Test: {@link PublisherAPIImpl#publish(PublisherConfig)}
420+
* When: Add different assets into a bundle, and send it
421+
* Should: Create all the files
422+
*/
423+
@Test
424+
@UseDataProvider("publishers")
425+
public void sendPushPublishBundle(final TestAsset testAsset)
426+
throws DotPublishingException, DotSecurityException, IOException, DotDataException, DotPublisherException {
427+
final Class<? extends Publisher> publisher = PushPublisher.class;
428+
429+
final Environment environment = new EnvironmentDataGen().nextPersisted();
430+
431+
final PushPublishingEndPoint publishingEndPoint = new PushPublishingEndPointDataGen()
432+
.environment(environment)
433+
.nextPersisted();
434+
435+
final FilterDescriptor filterDescriptor = new FilterDescriptorDataGen().nextPersisted();
436+
437+
final PushPublisherConfig config = new PushPublisherConfig();
438+
config.setPublishers(list(publisher));
439+
config.setOperation(PublisherConfig.Operation.PUBLISH);
440+
config.setLuceneQueries(list());
441+
config.setId("sendPushPublishBundle_" + System.currentTimeMillis());
442+
443+
final Bundle bundle = new BundleDataGen()
444+
.pushPublisherConfig(config)
445+
.addAssets(list(testAsset.asset))
446+
.filter(filterDescriptor)
447+
.nextPersisted();
448+
449+
final BundleFactoryImpl bundleFactory = new BundleFactoryImpl();
450+
bundleFactory.saveBundleEnvironment(bundle, environment);
451+
452+
final Collection<Object> dependencies = new HashSet<>();
453+
dependencies.addAll(testAsset.expectedInBundle);
454+
455+
createLanguageVariableIfNeeded();
456+
addLanguageVariableDependencies(dependencies, testAsset.addLanguageVariableDependencies);
457+
458+
final PublisherAPIImpl publisherAPI = new PublisherAPIImpl();
459+
460+
final PublishAuditStatus publishAuditStatus = new PublishAuditStatus(bundle.getId());
461+
462+
final PublishAuditHistory publishAuditHistory = new PublishAuditHistory();
463+
publishAuditStatus.setStatusPojo(publishAuditHistory);
464+
465+
PublishAuditAPI.getInstance().insertPublishAuditStatus(publishAuditStatus);
466+
467+
final File tempFile = com.dotmarketing.util.FileUtil
468+
.createTemporaryFile("sendPushPublishBundle_");
469+
470+
HttpServer httpServer = createHttpServer(tempFile);
471+
472+
try {
473+
httpServer.start();
474+
final PublishStatus publish = publisherAPI.publish(config);
475+
File bundleRoot = publish.getOutputFiles().get(0);
476+
477+
assertTrue(tempFile.exists());
478+
assertTrue(tempFile.length() > 0);
479+
424480
final File extractHere = new File(bundleRoot.getParent() + File.separator + config.getName());
425-
extractTarArchive(bundleRoot, extractHere);
426-
bundleRoot = extractHere;
481+
extractTarArchive(tempFile, extractHere);
482+
assertBundle(testAsset, dependencies, extractHere);
483+
} finally {
484+
httpServer.stop(0);
427485
}
486+
}
428487

429-
assertBundle(testAsset, dependencies, bundleRoot);
488+
private HttpServer createHttpServer(File tempFile) throws IOException {
489+
final HttpServer httpServer = HttpServer.create(new InetSocketAddress("127.0.0.1", 8080), 0);
490+
491+
httpServer.createContext("/api/bundlePublisher/publish", exchange -> {
492+
final InputStream responseBody = exchange.getRequestBody();
493+
FileUtils.copyInputStreamToFile(responseBody, tempFile);
494+
exchange.sendResponseHeaders( HttpURLConnection.HTTP_OK, 0);
495+
exchange.close();
496+
});
497+
498+
return httpServer;
430499
}
431500

432501
private void assertBundle(TestAsset testAsset, Collection<Object> dependencies, File bundleRoot)
@@ -487,18 +556,6 @@ private List<String> getDifferences(long numberFilesExpected, int numberFiles, L
487556
return differences;
488557
}
489558

490-
private static class TestCase {
491-
Class<? extends Publisher> publisher;
492-
TestAsset asset;
493-
494-
public TestCase(
495-
final Class<? extends Publisher> publisher,
496-
TestAsset asset) {
497-
498-
this.publisher = publisher;
499-
this.asset = asset;
500-
}
501-
}
502559

503560
public static List<Contentlet> getLanguageVariables() throws DotDataException, DotSecurityException {
504561
final User systemUser = APILocator.systemUser();

‎dotCMS/src/main/java/com/dotcms/publisher/pusher/PushPublisher.java‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,9 @@
3434
import com.dotcms.publishing.Publisher;
3535
import com.dotcms.publishing.PublisherConfig;
3636
import com.dotcms.publishing.PublisherConfig.DeliveryStrategy;
37+
import com.dotcms.publishing.output.BundleOutput;
38+
import com.dotcms.publishing.output.DirectoryBundleOutput;
39+
import com.dotcms.publishing.output.TarGzipBundleOutput;
3740
import com.dotcms.repackage.org.apache.commons.httpclient.HttpStatus;
3841
import com.dotcms.repackage.org.apache.commons.io.FileUtils;
3942
import com.dotcms.rest.ResourceResponse;
@@ -148,16 +151,7 @@ public PublisherConfig process ( final PublishStatus status ) throws DotPublishi
148151
File bundleRoot = BundlerUtil.getBundleRoot(this.config.getName(), false);
149152
ArrayList<File> list = new ArrayList<File>(1);
150153
list.add(bundleRoot);
151-
File bundleFile = new File(bundleRoot+File.separator+".."+File.separator+this.config.getId()+".tar.gz");
152-
153-
// If the tar.gz doesn't exist or if it the first try to push bundle
154-
// we need to compress the bundle folder into the tar.gz file.
155-
if (!bundleFile.exists() || !pubAuditAPI.isPublishRetry(config.getId())) {
156-
PushUtils.compressFiles(list, bundleFile, bundleRoot.getAbsolutePath());
157-
} else {
158-
Logger.info(this, "Retrying bundle: " + config.getId()
159-
+ ", we don't need to compress bundle again");
160-
}
154+
File bundleFile = new File(bundleRoot + ".tar.gz");
161155

162156
List<Environment> environments = APILocator.getEnvironmentAPI().findEnvironmentsByBundleId(this.config.getId());
163157

@@ -559,4 +553,8 @@ private void updateJobDataMap(DeliveryStrategy deliveryStrategy) {
559553
}
560554
}
561555

556+
@Override
557+
public BundleOutput createBundleOutput() throws IOException {
558+
return new TarGzipBundleOutput(config);
559+
}
562560
}

‎dotCMS/src/main/java/com/dotcms/publishing/PublisherAPIImpl.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -137,10 +137,10 @@ final public PublishStatus publish ( PublisherConfig config, PublishStatus statu
137137
Logger.info(this, "Retrying bundle: " + config.getId()
138138
+ ", we don't need to run bundlers again");
139139
}
140-
141-
publisher.process(status);
142140
}
143141

142+
publisher.process(status);
143+
144144
config.setBundlers(confBundlers);
145145

146146
//Triggering event listener when the publishing process ends

0 commit comments

Comments
 (0)