Skip to content

Commit dba5f37

Browse files
qingfengxiaqingfengxia
authored
Merge pull request #15 from qingfengxia/dev
add technical background for parallel-preprocessing
2 parents abd04d8 + 23b2833 commit dba5f37

31 files changed

Lines changed: 3741 additions & 3575 deletions

‎.github/workflows/conda-ci.yml‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
name: parallel-preprocessor-conda
2+
# https://github.com/marketplace/actions/setup-miniconda
3+
on: [push]
4+
5+
jobs:
6+
ppp-conda-ci:
7+
name: Ex1 (${{ matrix.python-version }}, ${{ matrix.os }})
8+
runs-on: ${{ matrix.os }}
9+
strategy:
10+
fail-fast: false
11+
matrix:
12+
os: ["ubuntu-latest", "windows-latest"]
13+
python-version: ["3.6", "3.7"]
14+
steps:
15+
- uses: actions/checkout@v2
16+
- uses: goanpeca/setup-miniconda@v1
17+
with:
18+
auto-update-conda: true
19+
python-version: ${{ matrix.python-version }}
20+
channels: conda-forge
21+
channel-priority: flexible
22+
activate-environment: foo # the default env
23+
#activate-environment: anaconda-client-env
24+
#environment-file: etc/example-environment.yml
25+
26+
# - name: Conda install dependencies
27+
# shell: bash -l {0}
28+
# run: |
29+
# conda info
30+
# conda install cmake
31+
# #only cmake installed from conda can find the conda install package?
32+
# conda install -c conda-forge occt=7.4
33+
34+
# - name: compile on windows
35+
# shell: bash -l {0}
36+
# run: |
37+
# mkdir condabuild
38+
# cd condabuild
39+
# cmake -G "NMake Makefiles" ..
40+
# cmake --build .
41+
# if: matrix.os == 'windows-latest'
42+
43+
# - name: compile on unix-like OS
44+
# shell: bash -l {0}
45+
# run: |
46+
# mkdir condabuild
47+
# cd condabuild
48+
# cmake ..
49+
# cmake --build .
50+
# if: matrix.os != 'windows-latest'
51+
52+
- name: build conda package on windows and Linux
53+
#shell: bash -l {0}
54+
run: |
55+
conda info
56+
conda install conda-build
57+
conda build ../recipe/meta.yaml
58+
59+
- name: test built package
60+
shell: bash -l {0}
61+
run: |
62+
conda install -c local ppp
63+
echo "start unit test in the folder:$(pwd)"
64+
if [ ! -f "run_all_tests.sh" ]; then cp ../scripts/run_all_tests.sh ./ ; fi
65+
bash run_all_tests.sh
66+
if: matrix.os != 'windows-latest'
67+
68+
- name: Create Release
69+
id: create_release
70+
uses: actions/create-release@v1 #https://github.com/actions/upload-release-asset
71+
env:
72+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
73+
with:
74+
tag_name: ${{ github.ref }}
75+
release_name: Release ${{ github.ref }}
76+
draft: false
77+
prerelease: false
78+
79+
- name: Upload Release Asset
80+
id: upload-release-asset
81+
uses: actions/upload-release-asset@v1
82+
env:
83+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
84+
with:
85+
upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
86+
asset_path: ./my-artifact.zip # todo: tar.gz file on Linux
87+
asset_name: my-artifact.zip
88+
asset_content_type: application/zip

‎.github/workflows/docker-ci.yml‎

Lines changed: 24 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: parallel-preprocessor-docker
1+
name: fedora-debian
22

33
on: [push, pull_request]
44

@@ -28,28 +28,39 @@ jobs:
2828
# compiler: gcc
2929
# compilerpp: g++
3030
# package_suffix: 'rpm'
31+
# docker-image: "centos:8"
32+
- os: 'debian-10'
33+
compiler: gcc
34+
compilerpp: g++
35+
package_suffix: 'deb'
36+
docker-image: "debian:10" # version 10 buster
3137
container:
32-
image: fedora:32
38+
image: ${{ matrix.docker-image }}
3339
steps:
34-
- name: 'install git'
35-
id: 'install-git'
40+
- name: 'install dependencies on fedora'
41+
id: 'install-dep-fedora'
3642
run: |
3743
yum install g++ gcc cmake make rpm-build git wget copr-cli -y && yum update -y
38-
39-
- uses: actions/checkout@v2
40-
# GitHub's actions/checkout requires git version 2.18 or later, if run inside docker
41-
#- name: Checkout submodules
42-
# uses: textbook/git-checkout-submodule-action@master
43-
44-
- name: 'install dependencies'
45-
id: 'install-dep'
46-
run: |
4744
yum install tbb tbb-devel freetype freetype-devel freeimage freeimage-devel \
4845
glew-devel SDL2-devel SDL2_image-devel glm-devel libXmu-devel libXi-devel \
4946
python3 python3-devel boost-devel -y
5047
yum install opencascade-foundation opencascade-modeling opencascade-ocaf \
5148
opencascade-visualization opencascade-devel -y
49+
if: matrix.os == 'fedora-31' || matrix.os == 'fedora-32'
50+
51+
- name: 'install dependencies on debian'
52+
id: 'install-dep-debian'
53+
run: |
54+
apt-get update
55+
apt-get install -y g++ cmake git gcc build-essential
56+
apt-get install -y libtbb*-dev libocct-*-dev occt-misc python3-dev python3-pybind11 libboost-dev
57+
apt-get install -y libx11-dev libxmu-dev libxi-dev
58+
if: matrix.os == 'debian-10'
5259

60+
- uses: actions/checkout@v2
61+
# GitHub's actions/checkout requires git version 2.18 or later, if run inside docker
62+
#- name: Checkout submodules
63+
# uses: textbook/git-checkout-submodule-action@master
5364

5465
- name: 'build package'
5566
id: 'build-package'

‎.github/workflows/github-ci.yml‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: parallel-preprocessor
1+
name: ubuntu-macos
22
# adapted from: https://github.com/ScottishCovidResponse/Covid19_EERAModel/blob/dev/.github/workflows/eera.yml
33
# Supported runners and hardware resources
44
# https://docs.github.com/en/free-pro-team@latest/actions/reference/specifications-for-github-hosted-runners#supported-runners-and-hardware-resources
@@ -19,17 +19,17 @@ jobs:
1919
matrix:
2020
include:
2121
- os: 'ubuntu-20.04'
22-
comiler: gcc
23-
comilerpp: g++
22+
compiler: gcc
23+
compilerpp: g++
2424
package_suffix: 'deb'
2525
- os: 'ubuntu-18.04'
26-
comiler: gcc
27-
comilerpp: g++
26+
compiler: gcc
27+
compilerpp: g++
2828
package_suffix: 'deb'
29-
# - os: 'macos-latest'
30-
# comiler: gcc
31-
# comilerpp: g++
32-
# package_suffix: 'dmg'
29+
- os: 'macos-latest'
30+
compiler: gcc
31+
compilerpp: g++
32+
package_suffix: 'dmg'
3333
steps:
3434
- uses: actions/checkout@v2
3535

@@ -39,7 +39,7 @@ jobs:
3939
#sudo apt-get upgrade
4040
sudo apt-get install -y g++ cmake clang cppcheck similarity-tester flawfinder
4141
sudo apt-get install -y lcov htmldoc poppler-utils graphviz doxygen
42-
sudo apt-get install -y libtbb*-dev libocct-*-dev occt-misc python3-dev python3-pybind11 freecad
42+
sudo apt-get install -y libtbb*-dev libocct-*-dev occt-misc python3-dev python3-pybind11 libboost-dev freecad
4343
sudo apt-get install -y libx11-dev libxmu-dev libxi-dev
4444
if: matrix.os == 'ubuntu-20.04'
4545
# ubuntu-20.04 has opencascade in official repo, PPA not needed, python3-pybind11 is also latest enough
@@ -49,15 +49,15 @@ jobs:
4949
sudo add-apt-repository ppa:freecad-maintainers/freecad-stable
5050
sudo apt-get update
5151
#sudo apt-get upgrade
52-
sudo apt-get install -y libtbb*-dev libocct-*-dev occt-misc python3-dev freecad
52+
sudo apt-get install -y libtbb*-dev libocct-*-dev occt-misc python3-dev freecad libboost-dev
5353
sudo apt-get install -y g++ cmake clang lcov cppcheck similarity-tester flawfinder clang-format clang-tidy
5454
sudo apt-get install -y libx11-dev libxmu-dev libxi-dev
5555
if: matrix.os == 'ubuntu-18.04'
5656

5757
- name: Compile on Linux (debug)
5858
# env:
59-
# CC: ${{ matrix.config.compiler }}
60-
# CXX: ${{ matrix.config.compilerpp }}
59+
# CC: ${{ matrix.compiler }}
60+
# CXX: ${{ matrix.compilerpp }}
6161
run: |
6262
git submodule update --init --recursive
6363
mkdir build

‎CMakeLists.txt‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -247,24 +247,25 @@ if (UNIX)
247247
if (${CMAKE_OS_NAME} STREQUAL "Debian" OR ${CMAKE_OS_NAME} STREQUAL "Ubuntu")
248248
# Tell CPack to generate a .deb package
249249
set(CPACK_GENERATOR "DEB")
250-
set(CPACK_PACKAGE_NAME "parallel-preprocessor_${CMAKE_OS_ID}-${CMAKE_OS_VERSION}")
251-
# -${CPACK_DEBIAN_PACKAGE_ARCHITECTURE} not defined variable
252-
string( TOLOWER "${PACKAGE_NAME}-${PACKAGE_VERSION_NAME}_${CMAKE_OS_ID}-${CMAKE_OS_VERSION}"
253-
CPACK_PACKAGE_FILE_NAME )
250+
set(CPACK_PACKAGE_NAME ${PACKAGE_NAME})
254251
endif()
255252
if (${CMAKE_OS_NAME} STREQUAL "RedHat" OR ${CMAKE_OS_NAME} STREQUAL "Fedora")
256253
set(CPACK_GENERATOR "RPM")
257-
set(CPACK_PACKAGE_NAME "parallel-preprocessor_${CMAKE_OS_ID}-${CMAKE_OS_VERSION}")
254+
set(CPACK_PACKAGE_NAME ${PACKAGE_NAME})
258255
endif()
259256
if (APPLE)
260257
set(CPACK_GENERATOR "DragNDrop")
261-
set(CPACK_PACKAGE_NAME "parallel-preprocessor_${CMAKE_OS_ID}-${CMAKE_OS_VERSION}")
258+
set(CPACK_PACKAGE_NAME ${PACKAGE_NAME})
262259
endif()
260+
261+
# -${CPACK_DEBIAN_PACKAGE_ARCHITECTURE} not defined variable
262+
string( TOLOWER "${PACKAGE_NAME}-${PACKAGE_VERSION_NAME}_${CMAKE_OS_ID}-${CMAKE_OS_VERSION}"
263+
CPACK_PACKAGE_FILE_NAME )
263264
endif()
264265
if(WIN32)
265266
# NSIS can be generated by opencascade and Qt dll are not bundled, so not quite working
266267
set(CPACK_GENERATOR "NSIS64") # zip 7z are other choices
267-
set(CPACK_PACKAGE_NAME "parallel-preprocessor-${CMAKE_OS_VERSION}_${CMAKE_OS_ID}")
268+
set(CPACK_PACKAGE_NAME ${PACKAGE_NAME})
268269
set(CPACK_PACKAGE_INSTALL_REGISTRY_KEY ${PACKAGE_NAME})
269270
endif()
270271

@@ -275,16 +276,18 @@ set(CPACK_DEBIAN_PACKAGE_MAINTAINER "Qingfeng Xia @ UKAEA")
275276
set(CPACK_PACKAGE_DESCRIPTION, "${CMAKE_CURRENT_SOURCE_DIR}/Readme.md")
276277
# Set a Package Version
277278
set(CPACK_PACKAGE_VERSION ${PACKAGE_VERSION_NAME})
278-
#set(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}")
279+
if (NOT APPLE)
280+
# this LICENSE file's end of line caused problem during dmg package generation on MacOS
279281
set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE")
282+
endif()
280283

281284
message(STATUS "CPACK_PACKAGE_FILE_NAME: ${CPACK_PACKAGE_FILE_NAME}")
282285

283286
################################################################
284287

285288
#set(CPACK_IGNORE_FILES "\.psd$;/\.git/;/backup/;\.#;/#;\.tar.gz$;/stage/;/build/;/condabuild/;\.diff$;\.DS_Store")
286-
set(CPACK_SOURCE_GENERATOR "TGZ")
287-
set(CPACK_SOURCE_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}")
289+
set(CPACK_SOURCE_GENERATOR "ZIP")
290+
set(CPACK_SOURCE_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${PACKAGE_VERSION_NAME}")
288291
set(CPACK_SOURCE_IGNORE_FILES ${CPACK_IGNORE_FILES})
289292

290293
#################################################################

‎Readme.md‎

Lines changed: 32 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,20 @@ Dr Andrew Davis of UKAEA, has contributed his technical insight, test geometrie
1616

1717
[**doxygen generated API documentation with wiki pages**](https://ukaea.github.io/parallel-preprocessor/site/doxygen-docs.html)
1818

19-
## Overview
19+
## Feature overview
2020

21-
Although this software aims to be a framework for more CAE/CAE preprocessing operatons, currently, this software provides only multi-threading geometry imprint and collision check, via command line user interface. This software has demonstrated faster and more controllable geometry imprinting on large geometry assemblies (10k+ parts) that is not possible on most existing CAD tools.
21+
This software aims to be a framework for more CAE/CAE **preprocessing operatons** for large geometry assemblies upto 1 millions parts, such as such as fusion reactor, areoplane, areo-engine as a whole, using high performance computation infrastructure like Exa-scale super-computer. see more [Technical Overview](./wiki/TechOverview.md) on why a parallel preprocessor is needed in the era of E-scale (10^18 FLops) computation.
2222

23-
Technical backgroud will be added later
23+
Currently, this software provides only multi-threading geometry imprint and collision check, via command line user interface. This software has demonstrated faster and more controllable geometry **collision-detection, imprinting** on **large geometry assemblies (10k+ parts)** that is not possible on most existing CAD tools.
24+
25+
![CPU usage of parallel-preprocessor using 64 threads on a 32-core CPU](./wiki/assets/ppp_multithreading_cpu_usage.png)
26+
27+
Screenshot for the CPU usage of parallel-preprocessor using 64 threads on a 32-core CPU (Source: Dr Andy Davis)
28+
29+
30+
## Future plan
31+
32+
[wiki/Roadmap.md](wiki/Roadmap.md): lists short-term and long-term plan, depends on funding status. Partially sponsoring this project is welcomed to enhance existing modules or develop new modules.
2433

2534
## Disclaimer
2635

@@ -32,39 +41,50 @@ According to the open source [license](./LICENSE), **there is no warranty for t
3241

3342
This project has been designed to be cross-platform, but only Linux is supported as the baseline platform.
3443

35-
+ Ubuntu as the primary/baseline development platform, with deb binary package generated
44+
+ Ubuntu latest LTS as the primary/baseline development platform, with deb binary package generated
45+
- Debian package should be achievable, since OpenCASCADE are available in official repository
3646

37-
+ Fedora, Centos can compile ppp from source, with OpenCascade 7.x source code or copr package.
47+
+ Fedora 30+ with OpenCascade 7.x package available from official repository, with rpm binary package generated.
3848

39-
+ Centos8 should work without much effort; Centos7 software stack is outdated, using docker/singularity instead.
49+
+ Compiling from source code for other Linux platforms is straight-forward, driven by cmake and cpack, guidance provided.
50+
- Centos8 should work without much effort, but OpenCASCADE must be compiled from source at first.
51+
- Centos7 software stack is outdated for compiler and cmake , using docker/singularity instead.
4052

4153
+ Windows 10 users are encouraged to use WSL with one of the supported Linux distributions, while guide to compile on Windows has been added.
4254

43-
+ MacOS should be possible through homebrew, and compiling instruction has been provided.
55+
+ MacOS compiling and packaging is done via homebrew, DragNDrop binary package is available.
4456

4557
Conda package and Linux native package for Ubuntu LTS may be available in the future, see [packaging.md](wiki/Packaging.md)
4658

4759
## Installation guide
4860

49-
### Download binary package
61+
**Note: user must install runtime dependencies (TBB, OpenCASCADE, etc, see Compile guide wiki pages for each platform) then install the downloaded binary package. Hint: if user have freecad installed, then all dependencies should have installed**
62+
63+
### Download (x86_64 architecture) binary packages
5064
Ubuntu deb package and fedora 30+ rpm package, conda packages for windows, it should be available to download on **github Release** for this public github. The unstable package build from the latest code on the main branch can be downloaded here <https://github.com/ukaea/parallel-preprocessor/releases/tag/dev>
5165

5266
**Note: choose the correct operation system, and the package is targeting at system-wide python3**
5367

54-
The package file has the name pattern: `parallel-preprocessor-<this_software_version>-dev_<OS name>-<OS version>.<package_suffix>`
55-
If your OS is not supported, you need to compile it by yourself, there is documentation for installation dependency and build for all major platforms.
68+
The package file has the name pattern: `parallel-preprocessor-<this_software_version>-dev_<OS name>-<OS version>.<package_suffix>` If your OS is not supported, you need to compile it by yourself, there is documentation for installation dependency and build for all major platforms.
5669

70+
`apt remove parallel-preprocessor`
71+
`dpkg -i parallel-preprocessor*.deb`
5772
[Download parallel-preprocessor for ubuntu version 18.04](https://github.com/ukaea/parallel-preprocessor/releases/download/dev/parallel-preprocessor-0.3-dev_ubuntu-18.04.deb)
5873

5974
[Download parallel-preprocessor for ubuntu version 20.04](https://github.com/ukaea/parallel-preprocessor/releases/download/dev/parallel-preprocessor-0.3-dev_ubuntu-20.04.deb)
6075

76+
[Download parallel-preprocessor for debian version 10](https://github.com/ukaea/parallel-preprocessor/releases/download/dev/parallel-preprocessor-0.3-dev_debian-10.deb)
77+
78+
6179
[Download parallel-preprocessor for fedora version 31](https://github.com/ukaea/parallel-preprocessor/releases/download/dev/parallel-preprocessor-0.3-dev_fedora-31.rpm)
6280

6381
[Download parallel-preprocessor for fedora version 32](https://github.com/ukaea/parallel-preprocessor/releases/download/dev/parallel-preprocessor-0.3-dev_fedora-32.rpm)
6482

6583
Coming soon:
6684
[Download parallel-preprocessor for MacOS 10.15](https://github.com/ukaea/parallel-preprocessor/releases/download/dev/parallel-preprocessor-0.3-dev_macos-latest.dmg)
6785

86+
Coming later: Conda package for Windows 10.
87+
6888
### Compile from source
6989
[wiki/BuildOnLinux.md](wiki/BuildOnLinux.md): Guide to install dependencies and compile on Linux (Centos, Fedora, Ubuntu), build instructions.
7090

@@ -105,13 +125,6 @@ If your OS is not supported, you need to compile it by yourself, there is docum
105125
### Guide to packaging
106126
[wiki/Packaging.md](wiki/Packaging.md) deb/rpm package generation by CPack, distribution strategy.
107127

108-
---
109-
110-
## Future plan
111-
112-
[wiki/Roadmap.md](wiki/Roadmap.md): lists short-term and long-term plan, depends on funding status. Partially sponsoring this project is welcomed to enhance existing modules or develop new modules.
113-
114-
115128
---
116129

117130
## License consideration
@@ -125,4 +138,5 @@ Note: the main branch of this repo is `main` not `master`.
125138

126139
## Acknowledgement
127140

128-
Funding source: STEP project in UKAEA <http://www.ccfe.ac.uk/step.aspx>
141+
Funding from August 2019 ~ April 2020: STEP project in UKAEA <http://www.ccfe.ac.uk/step.aspx>
142+

‎src/CMakeLists.txt‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -123,9 +123,8 @@ endif(PPP_USE_PYTHON)
123123

124124

125125
############################################################
126-
# unit test module
126+
# unit test module, PropertyContainer is header only
127127
############################################################
128128
if(PPP_USE_TEST)
129-
add_subdirectory(test)
130129
add_subdirectory (PropertyContainer) # Catch2 must be add() before this subdirectionary
131130
endif()

‎src/Geom/GeomTests/CMakeLists.txt‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,19 @@ if(OCC_FOUND)
2020
target_link_libraries(geom_tests ${OCC_LIBS}) # list should be used without quote
2121
endif()
2222

23+
####################### standalone app #####################
24+
add_executable(UniqueIdValidator "UniqueIdValidator.cpp") #
25+
target_link_libraries(UniqueIdValidator ${OCC_LIBS})
26+
27+
install(TARGETS UniqueIdValidator
28+
RUNTIME DESTINATION bin
29+
COMPONENT applications)
30+
2331
##################### standalone app ###########################
2432
if (${PPP_USE_QT})
2533
# relies on QT, but user may not enable QT GUI
2634
add_executable(OccBVHdemo "OccBVHdemo.cpp") #
27-
target_link_libraries(OccBVHdemo ${OCC_LIBS}) # compiled but does not work with OCCT 7.3
35+
target_link_libraries(OccBVHdemo ${OCC_LIBS})
2836
target_link_libraries(OccBVHdemo ${OCC_VIZ_LIBRARIES}) # list should be used without quote
2937
install(TARGETS OccBVHdemo
3038
RUNTIME DESTINATION bin

0 commit comments

Comments
 (0)