Skip to content

Commit 88fc388

Browse files
jeanphi-baconnaisjeanphi-baconnais
authored
✨ Add Scheduler to refresh data (#58)
* ✨ Add Scheduler to refresh data every week
1 parent 67e42cd commit 88fc388

18 files changed

Lines changed: 387 additions & 226 deletions

‎README.md‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,13 @@ An interface made with Javelit is available.
3939
![zenika-open-source-dashboard](./docs/images/zenika-open-source-dashboard.png)
4040

4141

42-
> 🎯 The last resource will be removed after implementing all features and will be replaced by schedules.
42+
## 🕒 Scheduled Synchronization
43+
44+
The application automatically synchronizes data from GitHub and GitLab using scheduled tasks configured in Quarkus:
45+
46+
- **Data Synchronization Schedule (`DataSyncSchedule.java`)**:
47+
- **Configuration**: `datasync.cron` property in `application.properties` (defaults to every Monday at 8:00 AM: `0 0 8 ? * MON`).
48+
- **Development Mode**: Configured to run every 10 minutes (`0 */10 * ? * *`).
4349

4450
## 📝 Local Setup
4551

‎src/main/java/fr/zenika/opensource/stats/config/FirestoreCollections.java‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,25 @@
11
package fr.zenika.opensource.stats.config;
22

3-
import org.eclipse.microprofile.config.ConfigProvider;
4-
53
public enum FirestoreCollections {
64
PROJECTS("projects"),
75
MEMBERS("members"),
86
STATS("stats"),
7+
PARAMS("params"),
98
;
109

10+
private static volatile String prefix = "";
11+
12+
public static void setPrefix(String prefix) {
13+
FirestoreCollections.prefix = prefix == null ? "" : prefix;
14+
}
15+
1116
private final String baseName;
1217

1318
FirestoreCollections(String baseName) {
1419
this.baseName = baseName;
1520
}
1621

1722
public String getValue() {
18-
String prefix = ConfigProvider.getConfig()
19-
.getOptionalValue("firestore.collection.prefix", String.class)
20-
.orElse("");
2123
return prefix + baseName;
2224
}
2325
}

‎src/main/java/fr/zenika/opensource/stats/config/GitHubClient.java‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
package fr.zenika.opensource.stats.config;
22

33
import jakarta.ws.rs.GET;
4+
import jakarta.ws.rs.HeaderParam;
45
import jakarta.ws.rs.Path;
56
import jakarta.ws.rs.PathParam;
67
import jakarta.ws.rs.QueryParam;
7-
import org.eclipse.microprofile.config.ConfigProvider;
88
import org.eclipse.microprofile.rest.client.annotation.ClientHeaderParam;
99
import org.eclipse.microprofile.rest.client.inject.RegisterRestClient;
1010

@@ -20,39 +20,39 @@
2020
@Path("/")
2121
public interface GitHubClient {
2222

23-
default String prepareToken() {
24-
String token = ConfigProvider.getConfig().getOptionalValue("github.token", String.class).orElse("");
25-
return "Bearer " + token;
26-
}
27-
2823
@GET
29-
@ClientHeaderParam(name = "Authorization", value = "{fr.zenika.opensource.stats.config.GitHubClient.prepareToken}")
3024
@Path("/orgs/{organizationName}")
31-
GitHubOrganization getOrgnizationByName(@PathParam("organizationName") String organizationName);
25+
GitHubOrganization getOrgnizationByName(
26+
@HeaderParam("Authorization") String authHeader,
27+
@PathParam("organizationName") String organizationName);
3228

3329
@GET
34-
@ClientHeaderParam(name = "Authorization", value = "{fr.zenika.opensource.stats.config.GitHubClient.prepareToken}")
3530
@Path("/orgs/{organizationName}/members")
36-
List<GitHubMember> getOrganizationMembers(@PathParam("organizationName") String organizationName,
31+
List<GitHubMember> getOrganizationMembers(
32+
@HeaderParam("Authorization") String authHeader,
33+
@PathParam("organizationName") String organizationName,
3734
@QueryParam("per_page") int perPage,
3835
@QueryParam("page") int page);
3936

4037
@GET
41-
@ClientHeaderParam(name = "Authorization", value = "{fr.zenika.opensource.stats.config.GitHubClient.prepareToken}")
4238
@Path("/user/{login}")
43-
GitHubMember getUserInformation(@PathParam("login") String login);
39+
GitHubMember getUserInformation(
40+
@HeaderParam("Authorization") String authHeader,
41+
@PathParam("login") String login);
4442

4543
@GET
46-
@ClientHeaderParam(name = "Authorization", value = "{fr.zenika.opensource.stats.config.GitHubClient.prepareToken}")
4744
@Path("/users/{login}/repos")
48-
List<GitHubProject> getReposForAnUser(@PathParam("login") String login,
45+
List<GitHubProject> getReposForAnUser(
46+
@HeaderParam("Authorization") String authHeader,
47+
@PathParam("login") String login,
4948
@QueryParam("per_page") int perPage,
5049
@QueryParam("page") int page);
5150

5251
@GET
53-
@ClientHeaderParam(name = "Authorization", value = "{fr.zenika.opensource.stats.config.GitHubClient.prepareToken}")
5452
@Path("/orgs/{organizationName}/repos")
55-
List<GitHubProject> getOrganizationProjects(@PathParam("organizationName") String organizationName,
53+
List<GitHubProject> getOrganizationProjects(
54+
@HeaderParam("Authorization") String authHeader,
55+
@PathParam("organizationName") String organizationName,
5656
@QueryParam("per_page") int perPage,
5757
@QueryParam("page") int page);
5858
}

‎src/main/java/fr/zenika/opensource/stats/config/GitHubClientHeadersFactory.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package fr.zenika.opensource.stats.config;
2+
3+
import org.eclipse.microprofile.rest.client.ext.ClientHeadersFactory;
4+
import jakarta.ws.rs.core.MultivaluedHashMap;
5+
import jakarta.ws.rs.core.MultivaluedMap;
6+
import jakarta.enterprise.context.ApplicationScoped;
7+
import jakarta.inject.Inject;
8+
import org.eclipse.microprofile.config.inject.ConfigProperty;
9+
import java.util.Optional;
10+
11+
@ApplicationScoped
12+
public class GitHubClientHeadersFactory implements ClientHeadersFactory {
13+
14+
@Inject
15+
@ConfigProperty(name = "github.token")
16+
Optional<String> githubToken;
17+
18+
@Override
19+
public MultivaluedMap<String, String> update(MultivaluedMap<String, String> incomingHeaders, MultivaluedMap<String, String> clientOutgoingHeaders) {
20+
MultivaluedMap<String, String> result = new MultivaluedHashMap<>();
21+
result.putSingle("Authorization", "Bearer " + githubToken.orElse(""));
22+
return result;
23+
}
24+
}

‎src/main/java/fr/zenika/opensource/stats/config/GitLabClient.java‎

Lines changed: 16 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,11 @@
11
package fr.zenika.opensource.stats.config;
22

33
import jakarta.ws.rs.GET;
4+
import jakarta.ws.rs.HeaderParam;
45
import jakarta.ws.rs.Path;
56
import jakarta.ws.rs.PathParam;
67
import jakarta.ws.rs.QueryParam;
78
import jakarta.ws.rs.core.Response;
8-
import org.eclipse.microprofile.config.ConfigProvider;
9-
import org.eclipse.microprofile.rest.client.annotation.ClientHeaderParam;
109
import org.eclipse.microprofile.rest.client.inject.RegisterRestClient;
1110

1211
import fr.zenika.opensource.stats.beans.gitlab.GitLabEvent;
@@ -19,39 +18,39 @@
1918
@Path("/")
2019
public interface GitLabClient {
2120

22-
default String prepareToken() {
23-
String token = ConfigProvider.getConfig().getOptionalValue("gitlab.token", String.class).orElse("");
24-
return "Bearer " + token;
25-
}
26-
2721
@GET
28-
@ClientHeaderParam(name = "Authorization", value = "{fr.zenika.opensource.stats.config.GitLabClient.prepareToken}")
2922
@Path("/users")
30-
List<GitLabMember> getUserInformations(@QueryParam("username") String username);
23+
List<GitLabMember> getUserInformations(
24+
@HeaderParam("Authorization") String authHeader,
25+
@QueryParam("username") String username);
3126

3227
@GET
33-
@ClientHeaderParam(name = "Authorization", value = "{fr.zenika.opensource.stats.config.GitLabClient.prepareToken}")
3428
@Path("/users/{id}")
35-
GitLabMember getUserInformationById(@PathParam("id") String id);
29+
GitLabMember getUserInformationById(
30+
@HeaderParam("Authorization") String authHeader,
31+
@PathParam("id") String id);
3632

3733
@GET
38-
@ClientHeaderParam(name = "Authorization", value = "{fr.zenika.opensource.stats.config.GitLabClient.prepareToken}")
3934
@Path("/users/{id}/projects")
40-
List<GitLabProject> getProjectsForAnUser(@PathParam("id") String id);
35+
List<GitLabProject> getProjectsForAnUser(
36+
@HeaderParam("Authorization") String authHeader,
37+
@PathParam("id") String id);
4138

4239
@GET
43-
@ClientHeaderParam(name = "Authorization", value = "{fr.zenika.opensource.stats.config.GitLabClient.prepareToken}")
4440
@Path("/users/{id}/events")
45-
List<GitLabEvent> getEventsForAnUser(@PathParam("id") String id,
41+
List<GitLabEvent> getEventsForAnUser(
42+
@HeaderParam("Authorization") String authHeader,
43+
@PathParam("id") String id,
4644
@QueryParam("after") String after,
4745
@QueryParam("before") String before,
4846
@QueryParam("per_page") int perPage,
4947
@QueryParam("page") int page);
5048

5149
@GET
52-
@ClientHeaderParam(name = "Authorization", value = "{fr.zenika.opensource.stats.config.GitLabClient.prepareToken}")
5350
@Path("/merge_requests")
54-
Response getMergeRequests(@QueryParam("author_id") String authorId,
51+
Response getMergeRequests(
52+
@HeaderParam("Authorization") String authHeader,
53+
@QueryParam("author_id") String authorId,
5554
@QueryParam("state") String state,
5655
@QueryParam("updated_after") String updatedAfter,
5756
@QueryParam("updated_before") String updatedBefore,

‎src/main/java/fr/zenika/opensource/stats/config/GitLabClientHeadersFactory.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package fr.zenika.opensource.stats.config;
2+
3+
import org.eclipse.microprofile.rest.client.ext.ClientHeadersFactory;
4+
import jakarta.ws.rs.core.MultivaluedHashMap;
5+
import jakarta.ws.rs.core.MultivaluedMap;
6+
import jakarta.enterprise.context.ApplicationScoped;
7+
import jakarta.inject.Inject;
8+
import org.eclipse.microprofile.config.inject.ConfigProperty;
9+
import java.util.Optional;
10+
11+
@ApplicationScoped
12+
public class GitLabClientHeadersFactory implements ClientHeadersFactory {
13+
14+
@Inject
15+
@ConfigProperty(name = "gitlab.token")
16+
Optional<String> gitlabToken;
17+
18+
@Override
19+
public MultivaluedMap<String, String> update(MultivaluedMap<String, String> incomingHeaders, MultivaluedMap<String, String> clientOutgoingHeaders) {
20+
MultivaluedMap<String, String> result = new MultivaluedHashMap<>();
21+
result.putSingle("Authorization", "Bearer " + gitlabToken.orElse(""));
22+
return result;
23+
}
24+
}

‎src/main/java/fr/zenika/opensource/stats/ressources/workflow/WorkflowRessources.java‎

Lines changed: 66 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package fr.zenika.opensource.stats.ressources.workflow;
22

3+
import io.quarkus.logging.Log;
34
import jakarta.enterprise.context.ApplicationScoped;
45
import jakarta.inject.Inject;
56
import jakarta.ws.rs.POST;
@@ -10,8 +11,10 @@
1011
import jakarta.ws.rs.core.Response;
1112

1213
import java.time.LocalDate;
14+
import org.eclipse.microprofile.config.inject.ConfigProperty;
1315
import java.util.List;
1416
import java.util.Set;
17+
import java.util.HashSet;
1518
import java.util.stream.Collectors;
1619

1720
import fr.zenika.opensource.stats.beans.CustomStatsContributionsUserByMonth;
@@ -39,13 +42,31 @@ public class WorkflowRessources {
3942
@Inject
4043
FirestoreServices firestoreServices;
4144

45+
@ConfigProperty(name = "organization.name")
46+
String organizationName;
47+
48+
private Set<String> existingMemberLogins = new HashSet<>();
49+
private Set<String> existingGitLabUsernames = new HashSet<>();
50+
51+
4252
@POST
4353
@Path("members/save")
4454
@Produces(MediaType.TEXT_PLAIN)
4555
public Response saveMembers() throws DatabaseException {
4656

4757
// Load current state
4858
List<Member> existingMembers = firestoreServices.getAllMembers();
59+
60+
// Save the logins/usernames of already present members before upserting new ones
61+
existingMemberLogins = existingMembers.stream()
62+
.filter(m -> m.getGitHubAccount() != null)
63+
.map(m -> m.getGitHubAccount().getLogin())
64+
.collect(Collectors.toSet());
65+
existingGitLabUsernames = existingMembers.stream()
66+
.filter(m -> m.getGitlabAccount() != null)
67+
.map(m -> m.getGitlabAccount().getUsername())
68+
.collect(Collectors.toSet());
69+
4970
List<GitHubMember> gitHubMembers = gitHubServices.getOrganizationMembersFromConfig();
5071

5172
// Build a set of current GitHub logins in the organization
@@ -94,7 +115,7 @@ public Response saveForkedProject() {
94115
@Produces(MediaType.TEXT_PLAIN)
95116
public Response savePersonalProjects() throws DatabaseException {
96117

97-
firestoreServices.deleteAllProjects();
118+
firestoreServices.deleteAllGitHubProjects();
98119

99120
List<Member> members = firestoreServices.getAllMembers();
100121

@@ -109,35 +130,57 @@ public Response savePersonalProjects() throws DatabaseException {
109130
return Response.ok().build();
110131
}
111132

133+
@POST
134+
@Path("organization-projects/save")
135+
@Produces(MediaType.TEXT_PLAIN)
136+
public Response saveOrganizationProjects() throws DatabaseException {
137+
firestoreServices.deleteAllGitHubOrganizationProjects();
138+
List<GitHubProject> gitHubProjects = gitHubServices.getOrganizationProjects(organizationName);
139+
for (GitHubProject project : gitHubProjects) {
140+
project.setSource("GitHub Organization");
141+
firestoreServices.createProject(project);
142+
}
143+
return Response.ok().build();
144+
}
145+
112146
@POST
113147
@Path("stats/save/{year}")
114148
@Produces(MediaType.TEXT_PLAIN)
115149
public Response saveStatsForYear(@PathParam("year") int year) throws DatabaseException {
116150

117151
int currentYear = LocalDate.now().getYear();
118-
boolean isCurrentYear = (year == currentYear);
119-
120-
if (isCurrentYear) {
121-
firestoreServices.deleteStatsBySourceForYear(year, "GitHub");
122-
firestoreServices.deleteStatsBySourceForYear(year, "GitLab");
123-
}
124152

125153
List<Member> zMembers = firestoreServices.getAllMembers();
126154

155+
if (existingMemberLogins.isEmpty() && existingGitLabUsernames.isEmpty()) {
156+
existingMemberLogins = zMembers.stream()
157+
.filter(m -> m.getGitHubAccount() != null)
158+
.map(m -> m.getGitHubAccount().getLogin())
159+
.collect(Collectors.toSet());
160+
existingGitLabUsernames = zMembers.stream()
161+
.filter(m -> m.getGitlabAccount() != null)
162+
.map(m -> m.getGitlabAccount().getUsername())
163+
.collect(Collectors.toSet());
164+
}
165+
127166
for (Member member : zMembers) {
128167
// GitHub
129168
if (member.getGitHubAccount() != null) {
130-
// For past years, skip if stats already exist
131-
if (!isCurrentYear && firestoreServices.hasStatsForMemberAndYear(member.getId(), year, "GitHub")) {
132-
System.out.println("⏭️ Skip GitHub information for " + member.getGitHubAccount().getLogin()
133-
+ " (already exists)");
169+
// For past years, skip if the person was already present in the organization.
170+
// Sync only for new members to save API quota and preserve existing history.
171+
boolean isPastYear = (year < currentYear);
172+
boolean isExistingMember = existingMemberLogins.contains(member.getGitHubAccount().getLogin());
173+
boolean shouldSkip = isPastYear && isExistingMember;
174+
175+
if (shouldSkip) {
176+
Log.info("⏭️ Skip GitHub information for " + member.getGitHubAccount().getLogin() + " (already exists in organization)");
134177
} else {
135-
System.out.print("🔎 Check GitHub information for " + member.getGitHubAccount().getLogin());
178+
Log.info("🔎 Check GitHub information for " + member.getGitHubAccount().getLogin() + " (" + year + ")");
136179
List<CustomStatsContributionsUserByMonth> stats = gitHubServices
137180
.getContributionsForTheCurrentYear(member.getGitHubAccount().getLogin(), year);
138181
List<StatsContribution> statsList = StatsMapper.mapGitHubStatisticsToStatsContributions(
139182
member, year, stats);
140-
System.out.println("... ✅");
183+
Log.info("✅ GitHub contributions synced for " + member.getGitHubAccount().getLogin() + " (" + year + ")");
141184

142185
if (!statsList.isEmpty()) {
143186
for (StatsContribution stat : statsList) {
@@ -149,17 +192,21 @@ public Response saveStatsForYear(@PathParam("year") int year) throws DatabaseExc
149192

150193
// GitLab
151194
if (member.getGitlabAccount() != null) {
152-
// For past years, skip if stats already exist
153-
if (!isCurrentYear && firestoreServices.hasStatsForMemberAndYear(member.getId(), year, "GitLab")) {
154-
System.out.println("⏭️ Skip GitLab information for " + member.getGitlabAccount().getUsername()
155-
+ " (already exists)");
195+
// For past years, skip if the person was already present in the organization.
196+
// Sync only for new members to save API quota and preserve existing history.
197+
boolean isPastYear = (year < currentYear);
198+
boolean isExistingMember = existingGitLabUsernames.contains(member.getGitlabAccount().getUsername());
199+
boolean shouldSkip = isPastYear && isExistingMember;
200+
201+
if (shouldSkip) {
202+
Log.info("⏭️ Skip GitLab information for " + member.getGitlabAccount().getUsername() + " (already exists in organization)");
156203
} else {
157-
System.out.print("🔎 Check GitLab information for " + member.getGitlabAccount().getUsername());
204+
Log.info("🔎 Check GitLab information for " + member.getGitlabAccount().getUsername() + " (" + year + ")");
158205
List<CustomStatsContributionsUserByMonth> stats = gitLabServices
159206
.getContributionsForTheCurrentYear(member.getGitlabAccount().getUsername(), year);
160207
List<StatsContribution> statsList = StatsMapper.mapGitLabStatisticsToStatsContributions(
161208
member, year, stats);
162-
System.out.println("... ✅");
209+
Log.info("✅ GitLab contributions synced for " + member.getGitlabAccount().getUsername() + " (" + year + ")");
163210

164211
if (!statsList.isEmpty()) {
165212
for (StatsContribution stat : statsList) {

0 commit comments

Comments
 (0)