Skip to content

Commit 8e7834a

Browse files
epipavepipav
authored
feat: package downloads Tinybird resources (#3183)
1 parent 5b3e565 commit 8e7834a

8 files changed

Lines changed: 232 additions & 5 deletions

‎services/libs/tinybird/datasources/packageDownloads.datasource‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
TAGS "Package downloads widget"
2+
3+
SCHEMA >
4+
`id` UInt64 `json:$.record.id`,
5+
`date` Date `json:$.record.date`,
6+
`insightsProjectId` String `json:$.record.insights_project_id` DEFAULT '',
7+
`repo` String `json:$.record.repository_url` DEFAULT '',
8+
`name` String `json:$.record.name` DEFAULT '',
9+
`ecosystem` String `json:$.record.ecosystem` DEFAULT '',
10+
`purl` String `json:$.record.purl` DEFAULT '',
11+
`dependentReposCount` UInt64 `json:$.record.dependent_repos_count` DEFAULT 0,
12+
`dependentPackagesCount` UInt64 `json:$.record.dependent_packages_count` DEFAULT 0,
13+
`dockerDependentsCount` UInt64 `json:$.record.docker_dependents_count` DEFAULT 0,
14+
`dockerDownloadsCount` UInt64 `json:$.record.docker_downloads_count` DEFAULT 0,
15+
`downloadsCount` UInt64 `json:$.record.downloads_count` DEFAULT 0,
16+
`createdAt` DateTime64(3) `json:$.record.created_at`,
17+
`updatedAt` DateTime64(3) `json:$.record.updated_at`
18+
19+
ENGINE ReplacingMergeTree
20+
ENGINE_SORTING_KEY insightsProjectId, date, ecosystem, repo, name
21+
ENGINE_VER updatedAt

‎services/libs/tinybird/pipes/activities_filtered.pipe‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ SQL >
44
SELECT id, timestamp, type, platform, memberId, organizationId, segmentId
55
FROM activities_with_relations_sorted_deduplicated_ds a
66
where
7-
segmentId = (SELECT id FROM segments_filtered)
7+
segmentId = (SELECT segmentId FROM segments_filtered)
88
{% if defined(startDate) %}
99
AND a.timestamp
1010
> {{ DateTime(startDate, description="Filter activity timestamp after", required=False) }}

‎services/libs/tinybird/pipes/activities_filtered_historical_cutoff.pipe‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ SQL >
77
SELECT id, timestamp, type, platform, memberId, organizationId, segmentId
88
FROM activities_with_relations_sorted_deduplicated_ds a
99
where
10-
segmentId = (SELECT id FROM segments_filtered)
10+
segmentId = (SELECT segmentId FROM segments_filtered)
1111
{% if defined(startDate) %}
1212
AND a.timestamp
1313
<= {{ DateTime(startDate, description="Filter activity timestamp after", required=False) }}

‎services/libs/tinybird/pipes/activities_filtered_retention.pipe‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ SQL >
77
SELECT id, timestamp, type, platform, memberId, organizationId, segmentId
88
FROM activities_with_relations_sorted_deduplicated_ds a
99
where
10-
segmentId = (SELECT id FROM segments_filtered)
10+
segmentId = (SELECT segmentId FROM segments_filtered)
1111
{% if defined(startDate) %}
1212
AND a.timestamp
1313
> {% if defined(granularity) and granularity == "daily" %}

‎services/libs/tinybird/pipes/package_metrics.pipe‎

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
NODE package_downloads_filtered
2+
SQL >
3+
4+
%
5+
SELECT *
6+
FROM packageDownloads p FINAL
7+
where
8+
insightsProjectId = (select insightsProjectId from segments_filtered)
9+
{% if defined(startDate) %}
10+
AND p.date > toDate(
11+
{{ DateTime(startDate, description="Filter package downloads after", required=False) }}
12+
)
13+
{% end %}
14+
{% if defined(endDate) %}
15+
AND p.date < toDate(
16+
{{ DateTime(endDate, description="Filter package downloads before", required=False) }}
17+
)
18+
{% end %}
19+
{% if defined(ecosystem) %}
20+
AND p.ecosystem
21+
= {{ String(ecosystem, description="Filter package download ecosystem", required=False) }}
22+
{% end %}
23+
{% if defined(name) %}
24+
AND p.name = {{ String(name, description="Filter by package name", required=False) }}
25+
{% end %}
26+
27+
28+
29+
NODE package_downloads_timeseries_bounds
30+
SQL >
31+
32+
%
33+
{% if defined(startDate) and not defined(endDate) %}
34+
select toDate({{ startDate }}) as actual_start_date, toDate(now()) as actual_end_date
35+
{% end %}
36+
{% if not defined(startDate) and defined(endDate) %}
37+
select
38+
min(package_downloads_filtered.date) as actual_start_date,
39+
toDate({{ endDate }}) as actual_end_date
40+
from package_downloads_filtered
41+
where package_downloads_filtered.date > '1980-01-01'
42+
{% end %}
43+
{% if not defined(startDate) and not defined(endDate) %}
44+
select
45+
min(toDate(package_downloads_filtered.date)) as actual_start_date,
46+
toDate(now()) as actual_end_date
47+
from package_downloads_filtered
48+
where package_downloads_filtered.date > '1980-01-01'
49+
{% end %}
50+
{% if defined(startDate) and defined(endDate) %}
51+
select toDate({{ startDate }}) as actual_start_date, toDate({{ endDate }}) as actual_end_date
52+
{% end %}
53+
54+
55+
56+
NODE package_downloads_generate_timeseries
57+
SQL >
58+
59+
%
60+
SELECT DISTINCT
61+
CASE
62+
WHEN {{ granularity }} = 'daily'
63+
THEN toDate(addDays(package_downloads_timeseries_bounds.actual_start_date, number))
64+
WHEN {{ granularity }} = 'weekly'
65+
THEN toStartOfWeek(addDays(package_downloads_timeseries_bounds.actual_start_date, number * 7))
66+
WHEN {{ granularity }} = 'monthly'
67+
THEN toStartOfMonth(addMonths(package_downloads_timeseries_bounds.actual_start_date, number))
68+
WHEN {{ granularity }} = 'quarterly'
69+
THEN toStartOfQuarter(addMonths(package_downloads_timeseries_bounds.actual_start_date, number * 3))
70+
WHEN {{ granularity }} = 'yearly'
71+
THEN toStartOfYear(addYears(package_downloads_timeseries_bounds.actual_start_date, number))
72+
END AS "startDate",
73+
CASE
74+
WHEN {{ granularity }} = 'daily'
75+
THEN toDate(addDays(actual_start_date, number))
76+
WHEN {{ granularity }} = 'weekly'
77+
THEN toDate(toStartOfWeek(addDays(actual_start_date, number * 7)) + INTERVAL 6 DAY)
78+
WHEN {{ granularity }} = 'monthly'
79+
THEN
80+
toDate(
81+
toStartOfMonth(addMonths(actual_start_date, number))
82+
+ INTERVAL 1 MONTH
83+
- INTERVAL 1 DAY
84+
)
85+
WHEN {{ granularity }} = 'quarterly'
86+
THEN
87+
toDate(
88+
toStartOfQuarter(addMonths(actual_start_date, number * 3))
89+
+ INTERVAL 3 MONTH
90+
- INTERVAL 1 DAY
91+
)
92+
WHEN {{ granularity }} = 'yearly'
93+
THEN
94+
toDate(
95+
toStartOfYear(addYears(actual_start_date, number))
96+
+ INTERVAL 1 YEAR
97+
- INTERVAL 1 DAY
98+
)
99+
END AS "endDate"
100+
FROM numbers(1000)
101+
CROSS JOIN
102+
(
103+
SELECT
104+
CASE
105+
WHEN {{ granularity }} = 'weekly'
106+
THEN toStartOfWeek(actual_start_date)
107+
WHEN {{ granularity }} = 'monthly'
108+
THEN toStartOfMonth(actual_start_date)
109+
WHEN {{ granularity }} = 'quarterly'
110+
THEN toStartOfQuarter(actual_start_date)
111+
WHEN {{ granularity }} = 'yearly'
112+
THEN toStartOfYear(actual_start_date)
113+
ELSE actual_start_date
114+
END AS actual_start_date,
115+
actual_end_date
116+
FROM package_downloads_timeseries_bounds
117+
) package_downloads_timeseries_bounds
118+
WHERE "startDate" >= actual_start_date AND "startDate" < actual_end_date
119+
120+
121+
122+
NODE package_downloads_initial_aggregation
123+
SQL >
124+
125+
%
126+
SELECT
127+
{% if defined(granularity) %} p.date, {% end %}
128+
sum(p.downloadsCount) as downloadsCount,
129+
sum(p.dockerDownloadsCount) as "dockerDownloadsCount",
130+
sum(p.dockerDependentsCount) as "dockerDependentsCount",
131+
sum(p.dependentPackagesCount) as "dependentPackagesCount",
132+
sum(p.dependentReposCount) as "dependentReposCount"
133+
FROM package_downloads_filtered p
134+
group by p.insightsProjectId {% if defined(granularity) %}, p.date {% end %}
135+
136+
137+
138+
NODE package_downloads_timeseries_merge
139+
SQL >
140+
141+
%
142+
{% set onlyContributions = False %}
143+
{% if defined(granularity) %}
144+
SELECT
145+
ds."startDate",
146+
ds."endDate",
147+
max(pdf.downloadsCount) AS "downloadsCount",
148+
max(pdf.dockerDownloadsCount) AS "dockerDownloadsCount",
149+
max(pdf.dockerDependentsCount) AS "dockerDependentsCount",
150+
max(pdf.dependentPackagesCount) AS "dependentPackagesCount",
151+
max(pdf.dependentReposCount) AS "dependentReposCount"
152+
FROM package_downloads_generate_timeseries ds
153+
LEFT JOIN
154+
package_downloads_initial_aggregation pdf
155+
ON CASE
156+
WHEN {{ granularity }} = 'daily'
157+
THEN toDate(pdf.date)
158+
WHEN {{ granularity }} = 'weekly'
159+
THEN toStartOfWeek(pdf.date)
160+
WHEN {{ granularity }} = 'monthly'
161+
THEN toStartOfMonth(pdf.date)
162+
WHEN {{ granularity }} = 'quarterly'
163+
THEN toStartOfQuarter(pdf.date)
164+
WHEN {{ granularity }} = 'yearly'
165+
THEN toStartOfYear(pdf.date)
166+
END
167+
= ds."startDate"
168+
GROUP BY ds."startDate", ds."endDate"
169+
order by ds."startDate"
170+
{% else %} SELECT 1
171+
{% end %}
172+
173+
174+
175+
NODE package_downloads_result
176+
SQL >
177+
178+
%
179+
{% if not defined(granularity) %} SELECT * FROM package_downloads_initial_aggregation
180+
{% else %} select * from package_downloads_timeseries_merge
181+
{% end %}
182+
183+

‎services/libs/tinybird/pipes/packages.pipe‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
NODE packages_0
2+
SQL >
3+
4+
%
5+
SELECT distinct p.repo, p.name, p.ecosystem
6+
FROM packageDownloads p
7+
where
8+
insightsProjectId = (select insightsProjectId from segments_filtered)
9+
{% if defined(search) %}
10+
AND (
11+
p.name
12+
ilike '%'
13+
|| {{ String(search, description="Filter package download repo", required=False) }}
14+
|| '%'
15+
OR p.ecosystem
16+
ilike '%'
17+
||
18+
{{ String(search, description="Filter package download ecosystem", required=False) }}
19+
|| '%'
20+
)
21+
{% end %}
22+
23+

‎services/libs/tinybird/pipes/pull_requests_filtered.pipe‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ SQL >
77
SELECT *
88
FROM pull_requests_analyzed pra
99
where
10-
pra.segmentId = (SELECT id FROM segments_filtered)
10+
pra.segmentId = (SELECT segmentId FROM segments_filtered)
1111
{% if defined(repo) %}
1212
AND pra.channel = {{ String(repo, description="Filter activity repo", required=False) }}
1313
{% end %}

‎services/libs/tinybird/pipes/segments_filtered.pipe‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
NODE segments_filtered_0
22
SQL >
33
%
4-
SELECT "segmentId" as id, repositories
4+
SELECT id as insightsProjectId, segmentId, repositories
55
FROM insightsProjects FINAL
66
where
77
insightsProjects.enabled

0 commit comments

Comments
 (0)