Skip to content

Commit 611a4da

Browse files
authored
Refactor swift mining pipeline for git deployment (#789)
* Refactor swift Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> * Refactor swift Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> * Refactor swift Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> * Refactor swift Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> * Refactor tests for swift Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> * Refactor tests for swift Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> * Change checkpoint frequency Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> * Remove checkpoint from swift mining Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> * Bump version Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com> --------- Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>
1 parent 389a298 commit 611a4da

5 files changed

Lines changed: 152 additions & 266 deletions

File tree

minecode_pipelines/miners/swift.py

Lines changed: 0 additions & 99 deletions
This file was deleted.

minecode_pipelines/pipelines/mine_swift.py

Lines changed: 27 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -20,38 +20,46 @@
2020
# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
2121
# Visit https://github.com/aboutcode-org/scancode.io for support and download.
2222

23-
from scanpipe.pipelines import Pipeline
2423
from scanpipe.pipes import federatedcode
25-
from minecode_pipelines import pipes
26-
from minecode_pipelines.pipes.swift import mine_and_publish_swift_packageurls
2724

25+
from minecode_pipelines.pipelines import MineCodeBasePipeline
26+
from minecode_pipelines.pipes.swift import mine_swift_packageurls
27+
from minecode_pipelines.pipes.swift import load_swift_package_urls
2828

29-
class MineSwift(Pipeline):
29+
30+
class MineSwift(MineCodeBasePipeline):
3031
"""
31-
Mine all packageURLs from a swift index and publish them to a FederatedCode repo.
32+
Pipeline to mine Swift packages and publish them to FederatedCode.
3233
"""
3334

35+
swift_index_repo_url = "https://github.com/SwiftPackageIndex/PackageList"
36+
3437
@classmethod
3538
def steps(cls):
3639
return (
3740
cls.check_federatedcode_eligibility,
38-
cls.mine_and_publish_swift_packageurls,
39-
cls.delete_cloned_repos,
41+
cls.create_federatedcode_working_dir,
42+
cls.fetch_federation_config,
43+
cls.clone_swift_index,
44+
cls.mine_and_publish_packageurls,
45+
cls.delete_working_dir,
4046
)
4147

42-
def check_federatedcode_eligibility(self):
43-
"""
44-
Check if the project fulfills the following criteria for
45-
pushing the project result to FederatedCode.
46-
"""
47-
federatedcode.check_federatedcode_configured_and_available(logger=self.log)
48+
def clone_swift_index(self):
49+
"""Clone the Cargo index Repo."""
50+
self.swift_index_repo = federatedcode.clone_repository(
51+
repo_url=self.swift_index_repo_url,
52+
clone_path=self.working_path / "swift-index",
53+
logger=self.log,
54+
)
4855

49-
def mine_and_publish_swift_packageurls(self):
50-
"""Mine swift package names from swift indexes or checkpoint."""
51-
self.repos = mine_and_publish_swift_packageurls(self.log)
56+
def packages_count(self):
57+
return len(self.swift_packages_urls) if self.swift_packages_urls else None
5258

53-
def delete_cloned_repos(self):
54-
pipes.delete_cloned_repos(
55-
repos=self.repos,
59+
def mine_packageurls(self):
60+
self.swift_packages_urls = load_swift_package_urls(swift_index_repo=self.swift_index_repo)
61+
self.log(f"Total Swift packages to process: {len(self.swift_packages_urls)}")
62+
return mine_swift_packageurls(
63+
packages_urls=self.swift_packages_urls,
5664
logger=self.log,
5765
)

0 commit comments

Comments
 (0)