Skip to content

Commit a77ef64

Browse files
erickgonzalezdsilvamfreddyDOTCMSfabrizzio-dotCMSnollymar
authored
Merge 21.05 into master (#20425)
* Upgrading webcomponents version to 0.0.25 * Issue 20329 osgi not working java 8 (#20360) * Switch byte-buddy version depending on java used to run * Remove echo * Issue 19646 improvment memory space consumption (#20370) * #19646 Improvment heap scape consumption generating a bundle * #19646 Testing * Change ee version in dependencies.properties * Revert DotSubmitterchange and validate empty file * Rever dependencies.properties change * run-all * Bump up mysql connector version to be able to run in Java 11 (#20373) * Bump up mysql connector version to be able to run in Java 11 * run-all * #20358 earlier releases had a metadata column which isn't used anymore… (#20372) * #20358 earlier releases had a metadata column which isnt used anymore and it needs to be removed. * #20358 doc test * run-all * run-all * adding doc * run-all * #20336 Improving task performance (#20374) * #20336 Improving task performance * run-all * run-all * Removing semicolon in oracle script * run-all * run-all * Adding missing semicolon * run-all * run-oracle Co-authored-by: Nollymar Longa <> * #20312 Upserts (#20367) * #20312 Upserts * run-all * #20312 sql merge clean up * #20312 fixing typos * run-all * run-all * run-all * #20312 removing empty class * #20312 format code * run-all * #20312 removing trash file * #20312 log debug * run-all * #20312 safety measure to prevent constrint violation when removing data chunks * run-all * #20312 typo * run-oracle * run-all * #20364 xml is not longer allowed on content resource (#20381) * #20364 xml is not longer allowed on content resource * #20364 adding comments * #20364 minor refactor * run-all (#20375) * Avoid Out of Memory with huge file (#20392) * #20313 making es read only monitor less noisy (#20398) * Not use RestClient as a singleton (#20385) * Not use RestClient as a singleton * Removing this.restClient variable * Closing Rest Client * Issue 20386 index discrepancies metadata folder remove task (#20396) * #20386 fixing index discrepancies + metaData dir * #20386 relocating log * #20083 Update full starter with inline editing (#20409) * #20420 adding the none version to validation type on saml (#20421) * #19646 Log error nicely (#20419) * #19646 Log error nicely * remove import * Rename AddFileNotPossibleException to FileCreationException Co-authored-by: Nollymar Longa <> Co-authored-by: Daniel Silva <daniel.silva@dotcms.com> Co-authored-by: Freddy Rodriguez <freddy0309@gmail.com> Co-authored-by: fabrizzio-dotCMS <fabrizzio@dotCMS.com> Co-authored-by: Fabrizzio Araya <37148755+fabrizzio-dotCMS@users.noreply.github.com> Co-authored-by: Nollymar Longa <nollymar.longa@dotcms.com> Co-authored-by: Jonathan <jonathan.sanchez@dotcms.com> Co-authored-by: Victor Alfaro <victor.alfaro@dotcms.com>
1 parent a4c7585 commit a77ef64

34 files changed

Lines changed: 958 additions & 146 deletions

‎bin/startup.bat‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,24 @@ set JAVA_OPTS=%JAVA_OPTS% -XX:+UseG1GC
6565

6666
set JAVA_OPTS=%JAVA_OPTS% -Djava.endorsed.dirs=%DOTCMS_HOME%/WEB-INF/endorsed_libs
6767

68+
for /f tokens^=2-5^ delims^=.-_^" %%j in ('java -fullversion 2^>^&1') do set "JAVA_VERSION=%%j%%k"
69+
echo JAVA_VERSION = %JAVA_VERSION%
70+
set BYTE_BUDDY_VERSION=1.9.0
71+
72+
if %JAVA_VERSION% LSS 110 (
73+
set BYTE_BUDDY_VERSION=1.6.12
74+
)
75+
76+
echo Using Byte-Buddy Version: %BYTE_BUDDY_VERSION%
77+
78+
echo BYTE_BUDDY folder: %CATALINA_HOME%\webapps\ROOT\WEB-INF\lib\byte-buddy-*
79+
80+
del %CATALINA_HOME%\webapps\ROOT\WEB-INF\lib\byte-buddy-*
81+
copy %CATALINA_HOME%\bin\byte-buddy\%BYTE_BUDDY_VERSION%\* %CATALINA_BASE%\webapps\ROOT\WEB-INF\lib\
82+
83+
6884
rem Set agent opts
69-
set JAVA_OPTS=%JAVA_OPTS% -javaagent:%DOTCMS_HOME%/WEB-INF/lib/byte-buddy-agent-1.9.0.jar
85+
set JAVA_OPTS=%JAVA_OPTS% -javaagent:%DOTCMS_HOME%/WEB-INF/lib/byte-buddy-agent-%BYTE_BUDDY_VERSION%.jar
7086

7187
rem Uncomment the next line if you want to enable JMX
7288
rem set JAVA_OPTS=%JAVA_OPTS% -Dcom.sun.management.jmxremote.port=7788 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Djava.endorsed.dirs=$DOTCMS_HOME/WEB-INF/endorsed_libs

‎bin/startup.sh‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,23 @@ JAVA_OPTS="$JAVA_OPTS -XX:MaxMetaspaceSize=512m -Xmx1G"
6767
# Set GC opts
6868
JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC"
6969

70+
JAVA_VERSION="$(java -version 2>&1 | grep -i version | cut -d'"' -f2 | cut -d'.' -f1-2)"
71+
72+
echo "JAVA_VERSION: $JAVA_VERSION"
73+
BYTE_BUDDY_VERSION="1.9.0"
74+
75+
# Lexicographic comparation
76+
if [[ (($JAVA_VERSION < 11.0)) ]]; then
77+
BYTE_BUDDY_VERSION="1.6.12";
78+
fi
79+
80+
echo "Using BYTE_BUDDY_VERSION: $BYTE_BUDDY_VERSION"
81+
82+
rm $DOTCMS_HOME/WEB-INF/lib/byte-buddy-*
83+
cp $TOMCAT_HOME/bin/byte-buddy/$BYTE_BUDDY_VERSION/* $DOTCMS_HOME/WEB-INF/lib/
84+
7085
# Set agent opts
71-
JAVA_OPTS="$JAVA_OPTS -javaagent:$DOTCMS_HOME/WEB-INF/lib/byte-buddy-agent-1.9.0.jar"
86+
JAVA_OPTS="$JAVA_OPTS -javaagent:$DOTCMS_HOME/WEB-INF/lib/byte-buddy-agent-$BYTE_BUDDY_VERSION.jar"
7287

7388
# Set encoding
7489
JAVA_OPTS="$JAVA_OPTS -Dsun.jnu.encoding=UTF-8"

‎dotCMS/build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,7 @@ dependencies {
183183

184184
starter group: 'com.dotcms', name: 'starter', version: 'empty_20210429', ext: 'zip'
185185
//Uncomment this line if you want to download the starter that comes with data
186-
//starter group: 'com.dotcms', name: 'starter', version: '20210429', ext: 'zip'
186+
//starter group: 'com.dotcms', name: 'starter', version: '20210512', ext: 'zip'
187187
testsStarter group: 'com.dotcms', name: 'starter', version: 'empty_20210429', ext: 'zip'
188188

189189
profiler group: 'glowroot-custom', name: 'glowroot-agent', version: '0.13.1'

‎dotCMS/dependencies.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -501,7 +501,7 @@ dependencies {
501501
transitive = false
502502
}
503503
//https://mvnrepository.com/artifact/mysql/mysql-connector-java
504-
providedCompile (group: 'mysql', name: 'mysql-connector-java', version: '8.0.18') {
504+
providedCompile (group: 'mysql', name: 'mysql-connector-java', version: '8.0.19') {
505505
transitive = false
506506
}
507507
//https://mvnrepository.com/artifact/com.microsoft.sqlserver/mssql-jdbc

‎dotCMS/src/curl-test/Content Resource.postman_collection.json‎

Lines changed: 126 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"info": {
3-
"_postman_id": "0773d5de-70a0-445f-862c-928bcdfa2dda",
3+
"_postman_id": "766b73d8-a8cb-4c5b-ac3f-d971f6712b6d",
44
"name": "Content Resource",
55
"description": "Content Resource test",
66
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
@@ -456,14 +456,137 @@
456456
}
457457
},
458458
"url": {
459-
"raw": "{{serverURL}}/api/content/",
459+
"raw": "{{serverURL}}/api/content/save/1",
460460
"host": [
461461
"{{serverURL}}"
462462
],
463463
"path": [
464464
"api",
465465
"content",
466-
""
466+
"save",
467+
"1"
468+
]
469+
}
470+
},
471+
"response": []
472+
},
473+
{
474+
"name": "XML is not allowed on Save",
475+
"event": [
476+
{
477+
"listen": "test",
478+
"script": {
479+
"exec": [
480+
"pm.test(\"Status code should be 400\", function () {",
481+
" pm.response.to.have.status(400);",
482+
"});",
483+
"",
484+
"",
485+
"",
486+
""
487+
],
488+
"type": "text/javascript"
489+
}
490+
}
491+
],
492+
"request": {
493+
"auth": {
494+
"type": "basic",
495+
"basic": [
496+
{
497+
"key": "password",
498+
"value": "admin",
499+
"type": "string"
500+
},
501+
{
502+
"key": "username",
503+
"value": "admin@dotcms.com",
504+
"type": "string"
505+
}
506+
]
507+
},
508+
"method": "POST",
509+
"header": [],
510+
"body": {
511+
"mode": "raw",
512+
"raw": "<content>\n <contentType>webPageContent</contentType>\n <languageId>1</languageId>\n <title>test content</title>\n <contentHost>demo.dotcms.com</contentHost>\n <body>test body</body>\n</content>",
513+
"options": {
514+
"raw": {
515+
"language": "xml"
516+
}
517+
}
518+
},
519+
"url": {
520+
"raw": "{{serverURL}}/api/content/save/1",
521+
"host": [
522+
"{{serverURL}}"
523+
],
524+
"path": [
525+
"api",
526+
"content",
527+
"save",
528+
"1"
529+
]
530+
}
531+
},
532+
"response": []
533+
},
534+
{
535+
"name": "XML is not allowed on Publish",
536+
"event": [
537+
{
538+
"listen": "test",
539+
"script": {
540+
"exec": [
541+
"pm.test(\"Status code should be 400\", function () {",
542+
" pm.response.to.have.status(400);",
543+
"});",
544+
"",
545+
"",
546+
"",
547+
""
548+
],
549+
"type": "text/javascript"
550+
}
551+
}
552+
],
553+
"request": {
554+
"auth": {
555+
"type": "basic",
556+
"basic": [
557+
{
558+
"key": "password",
559+
"value": "admin",
560+
"type": "string"
561+
},
562+
{
563+
"key": "username",
564+
"value": "admin@dotcms.com",
565+
"type": "string"
566+
}
567+
]
568+
},
569+
"method": "PUT",
570+
"header": [],
571+
"body": {
572+
"mode": "raw",
573+
"raw": "<java.util.PriorityQueue serialization=\"custom\">\n <unserializable-parents/>\n <java.util.PriorityQueue>\n <default>\n <size>2</size>\n <comparator class=\"org.apache.commons.beanutils.BeanComparator\">\n <property>databaseMetaData</property>\n <comparator class=\"java.util.Collections$ReverseComparator\"/>\n </comparator>\n </default>\n <int>3</int>\n <com.sun.rowset.JdbcRowSetImpl serialization=\"custom\">\n <javax.sql.rowset.BaseRowSet>\n <default>\n <concurrency>1008</concurrency>\n <escapeProcessing>true</escapeProcessing>\n <fetchDir>1000</fetchDir>\n <fetchSize>0</fetchSize>\n <isolation>2</isolation>\n <maxFieldSize>0</maxFieldSize>\n <maxRows>0</maxRows>\n <queryTimeout>0</queryTimeout>\n <readOnly>true</readOnly>\n <rowSetType>1004</rowSetType>\n <showDeleted>false</showDeleted>\n <dataSource>jdbc/dotCMSPool</dataSource>\n <params/>\n </default>\n </javax.sql.rowset.BaseRowSet>\n <com.sun.rowset.JdbcRowSetImpl>\n <default>\n <iMatchColumns>\n <int>-1</int>\n <int>-1</int>\n <int>-1</int>\n <int>-1</int>\n <int>-1</int>\n <int>-1</int>\n <int>-1</int>\n <int>-1</int>\n <int>-1</int>\n <int>-1</int>\n </iMatchColumns>\n <strMatchColumns>\n <string>foo</string>\n <null/>\n <null/>\n <null/>\n <null/>\n <null/>\n <null/>\n <null/>\n <null/>\n <null/>\n </strMatchColumns>\n </default>\n </com.sun.rowset.JdbcRowSetImpl>\n </com.sun.rowset.JdbcRowSetImpl>\n <com.sun.rowset.JdbcRowSetImpl reference=\"../com.sun.rowset.JdbcRowSetImpl\"/>\n </java.util.PriorityQueue>\n</java.util.PriorityQueue>",
574+
"options": {
575+
"raw": {
576+
"language": "xml"
577+
}
578+
}
579+
},
580+
"url": {
581+
"raw": "{{serverURL}}/api/content/publish/1",
582+
"host": [
583+
"{{serverURL}}"
584+
],
585+
"path": [
586+
"api",
587+
"content",
588+
"publish",
589+
"1"
467590
]
468591
}
469592
},

‎dotCMS/src/integration-test/java/com/dotcms/MainSuite.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@
9696
import com.dotmarketing.startup.runonce.Task210319CreateStorageTableTest;
9797
import com.dotmarketing.startup.runonce.Task210321RemoveOldMetadataFilesTest;
9898
import com.dotmarketing.startup.runonce.Task210506UpdateStorageTableTest;
99+
import com.dotmarketing.startup.runonce.Task210510UpdateStorageTableDropMetadataColumnTest;
99100
import com.dotmarketing.util.ConfigTest;
100101
import com.dotmarketing.util.HashBuilderTest;
101102
import com.dotmarketing.util.TestConfig;
@@ -413,6 +414,7 @@
413414
FileAssetTemplateUtilTest.class,
414415
SiteSearchJobImplTest.class,
415416
Task210506UpdateStorageTableTest.class,
417+
Task210510UpdateStorageTableDropMetadataColumnTest.class,
416418
StaticPushPublishBundleGeneratorTest.class
417419
})
418420
public class MainSuite {

‎dotCMS/src/integration-test/java/com/dotcms/content/elasticsearch/business/ESMappingAPITest.java‎

Lines changed: 17 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,11 @@
22

33
import static com.dotcms.content.elasticsearch.business.ESMappingAPIImpl.INCLUDE_DOTRAW_METADATA_FIELDS;
44
import static com.dotcms.content.elasticsearch.business.ESMappingAPIImpl.INDEX_DOTRAW_METADATA_FIELDS;
5+
import static com.dotcms.content.elasticsearch.business.ESMappingAPIImpl.NO_METADATA;
56
import static com.dotcms.content.elasticsearch.business.ESMappingAPIImpl.TEXT;
67
import static com.dotcms.content.elasticsearch.business.ESMappingAPIImpl.WRITE_METADATA_ON_REINDEX;
78
import static com.dotcms.datagen.TestDataUtils.getCommentsLikeContentType;
9+
import static com.dotcms.datagen.TestDataUtils.getFileAssetContent;
810
import static com.dotcms.datagen.TestDataUtils.getMultipleImageBinariesContent;
911
import static com.dotcms.datagen.TestDataUtils.getNewsLikeContentType;
1012
import static com.dotcms.datagen.TestDataUtils.relateContentTypes;
@@ -40,6 +42,7 @@
4042
import com.dotcms.datagen.FileAssetDataGen;
4143
import com.dotcms.datagen.SiteDataGen;
4244
import com.dotcms.datagen.TestDataUtils;
45+
import com.dotcms.datagen.TestDataUtils.TestFile;
4346
import com.dotcms.util.CollectionsUtils;
4447
import com.dotcms.util.IntegrationTestInitService;
4548
import com.dotmarketing.beans.Host;
@@ -450,7 +453,7 @@ public void test_toMap_binary_field_shouldSuccess() throws Exception {
450453
assertEquals(320, contentletMap.get("metadata.width"));
451454
assertEquals(235, contentletMap.get("metadata.height"));
452455
assertEquals(true, contentletMap.get("metadata.isimage"));
453-
assertTrue( contentletMap.get("metadata.content").toString().trim().isEmpty());
456+
assertTrue( contentletMap.get("metadata.content").toString().trim().equals(NO_METADATA));
454457

455458
}
456459

@@ -495,29 +498,24 @@ public void Test_toMap_Metadata_dotRaw() {
495498
//Test that with the dotRaw fields generated are part of the list of inclusions
496499
Assert.assertTrue(includedDotRawFields.containsAll(dotRawMetaList));
497500

498-
//Now lets set an empty list to force skipping the defaults
499-
Config.setProperty(INCLUDE_DOTRAW_METADATA_FIELDS, "");
500-
final Map<String, Object> contentletMapIncludingNone = esMappingAPI
501-
.toMap(multipleBinariesContent);
501+
final Contentlet fileAssetContent = getFileAssetContent(true, 1L, TestFile.PDF);
502+
final Map<String, Object> contentletMapCustomInclude = esMappingAPI
503+
.toMap(fileAssetContent);
502504

503-
//Now lets get the list of metadata keys
504-
final List<String> dotRawMetaListForceNoneExclusion = contentletMapIncludingNone.keySet()
505-
.stream()
506-
.filter(s -> s.startsWith("metadata") && s.endsWith("dotraw"))
507-
.collect(Collectors.toList());
505+
assertTrue(contentletMapCustomInclude.containsKey("metadata.name"));
506+
assertTrue(contentletMapCustomInclude.containsKey("metadata.name_dotraw"));
508507

509-
Assert.assertTrue(dotRawMetaListForceNoneExclusion.isEmpty());
508+
assertTrue(contentletMapCustomInclude.containsKey("metadata.path"));
509+
assertTrue(contentletMapCustomInclude.containsKey("metadata.path_dotraw"));
510510

511-
//Now lets set a list with entries to exclude
512-
Config.setProperty(INCLUDE_DOTRAW_METADATA_FIELDS, "isImage,content");
513-
final Map<String, Object> contentletMapCustomInclude = esMappingAPI
514-
.toMap(multipleBinariesContent);
511+
assertTrue(contentletMapCustomInclude.containsKey("metadata.title"));
512+
assertTrue(contentletMapCustomInclude.containsKey("metadata.title_dotraw"));
515513

516-
assertTrue(contentletMapCustomInclude.containsKey("metadata.isimage"));
517-
assertTrue(contentletMapCustomInclude.containsKey("metadata.isimage_dotraw"));
514+
assertTrue(contentletMapCustomInclude.containsKey("metadata.moddate"));
515+
assertTrue(contentletMapCustomInclude.containsKey("metadata.moddate_dotraw"));
518516

519-
assertTrue(contentletMapCustomInclude.containsKey("metadata.content"));
520-
assertTrue(contentletMapCustomInclude.containsKey("metadata.content_dotraw"));
517+
assertTrue(contentletMapCustomInclude.containsKey("metadata.filesize"));
518+
assertTrue(contentletMapCustomInclude.containsKey("metadata.filesize_dotraw"));
521519

522520
//Test disconnecting the dot raw fields generation
523521
Config.setProperty(INDEX_DOTRAW_METADATA_FIELDS, false);

‎dotCMS/src/integration-test/java/com/dotcms/publishing/PublisherAPIImplTest.java‎

Lines changed: 35 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package com.dotcms.publishing;
22

3+
import com.dotcms.contenttype.model.field.Field;
4+
import com.dotcms.contenttype.model.field.TextField;
35
import com.dotcms.contenttype.model.type.ContentType;
46
import com.dotcms.contenttype.transform.contenttype.StructureTransformer;
57
import com.dotcms.datagen.*;
@@ -49,6 +51,7 @@
4951
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
5052
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
5153
import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
54+
import org.elasticsearch.index.fielddata.FieldData;
5255
import org.jetbrains.annotations.Nullable;
5356
import org.junit.AfterClass;
5457
import org.junit.BeforeClass;
@@ -131,7 +134,8 @@ public static Object[] publishers() throws Exception {
131134
getLinkWithDependencies(),
132135
getWorkflowWithDependencies(),
133136
getLanguageWithDependencies(),
134-
getRuleWithDependencies()
137+
getRuleWithDependencies(),
138+
getContentWithSeveralVersions()
135139
);
136140
final List<Class<? extends Publisher>> publishers = list(
137141
GenerateBundlePublisher.class,
@@ -143,13 +147,42 @@ public static Object[] publishers() throws Exception {
143147
for (final Class<? extends Publisher> publisher : publishers) {
144148
for (TestAsset asset : assets) {
145149
cases.add(new TestCase(publisher, asset));
146-
cases.add(new TestCase(publisher, asset));
147150
}
148151
}
149152

150153
return cases.toArray();
151154
}
152155

156+
private static TestAsset getContentWithSeveralVersions() throws DotDataException, DotSecurityException {
157+
final Host host = new SiteDataGen().nextPersisted();
158+
159+
final Field textField = new FieldDataGen().type(TextField.class).next();
160+
final ContentType contentType = new ContentTypeDataGen()
161+
.field(textField)
162+
.host(host)
163+
.nextPersisted();
164+
165+
final Contentlet liveVersion = new ContentletDataGen(contentType)
166+
.setProperty(textField.variable(), "Live versions")
167+
.host(host)
168+
.nextPersisted();
169+
170+
ContentletDataGen.publish(liveVersion);
171+
172+
final Contentlet workingVersion = ContentletDataGen.checkout(liveVersion);
173+
workingVersion.setStringProperty(textField.variable(), "Working versions");
174+
ContentletDataGen.checkin(workingVersion);
175+
176+
final WorkflowScheme systemWorkflowScheme = APILocator.getWorkflowAPI()
177+
.findSystemWorkflowScheme();
178+
179+
final Language defaultLanguage = APILocator.getLanguageAPI().getDefaultLanguage();
180+
181+
return new TestAsset(workingVersion,
182+
set(host, systemWorkflowScheme, contentType, liveVersion, defaultLanguage),
183+
"/bundlers-test/contentlet/contentlet/contentlet.content.xml");
184+
}
185+
153186
private static TestAsset getRuleWithDependencies() {
154187
final Host host = new SiteDataGen().nextPersisted();
155188

‎dotCMS/src/integration-test/java/com/dotcms/storage/StoragePersistenceAPITest.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -345,7 +345,6 @@ public static Object[] getRandomTestCases() throws Exception{
345345
* @throws IOException
346346
* @throws NoSuchAlgorithmException
347347
*/
348-
@Ignore //Ignored since it's failing on mySQL with Packet for query is too large (4,194,425 > 4,194,304). You can change this value on the server by setting the 'max_allowed_packet' variable.
349348
@Test
350349
@UseDataProvider("getLargeFileTestCases")
351350
public void Test_Push_Large_File(final TestCase testCase)

0 commit comments

Comments
 (0)