Skip to content

Commit 3c1c9a5

Browse files
OliverRietmannOliverRietmann
committed
Add __restrict__ types
1 parent 2877f3a commit 3c1c9a5

9 files changed

Lines changed: 88 additions & 18 deletions

‎GPU/GPUTracking/SectorTracker/GPUTPCBaseTrackParam.h‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,45 @@ struct BaseCRTP { };
2727
template <template <template <class> class> class S>
2828
struct BaseCRTP<S, wrapper::value> {
2929
using Derived = S<wrapper::value>;
30+
3031
constexpr operator S<wrapper::reference>() { return {{}, static_cast<Derived*>(this)->mX, static_cast<Derived*>(this)->mC, static_cast<Derived*>(this)->mZOffset, static_cast<Derived*>(this)->mP}; };
32+
constexpr operator S<wrapper::reference_restrict>() { return {{}, static_cast<Derived*>(this)->mX, static_cast<Derived*>(this)->mC, static_cast<Derived*>(this)->mZOffset, static_cast<Derived*>(this)->mP}; };
33+
3134
constexpr operator S<wrapper::const_reference>() const { return {{}, static_cast<const Derived*>(this)->mX, static_cast<const Derived*>(this)->mC, static_cast<const Derived*>(this)->mZOffset, static_cast<const Derived*>(this)->mP}; };
35+
constexpr operator S<wrapper::const_reference_restrict>() const { return {{}, static_cast<const Derived*>(this)->mX, static_cast<const Derived*>(this)->mC, static_cast<const Derived*>(this)->mZOffset, static_cast<const Derived*>(this)->mP}; };
3236
};
3337

3438
template <template <template <class> class> class S>
3539
struct BaseCRTP<S, wrapper::reference> {
3640
using Derived = S<wrapper::reference>;
41+
42+
constexpr operator S<wrapper::reference_restrict>() { return {{}, static_cast<Derived*>(this)->mX, static_cast<Derived*>(this)->mC, static_cast<Derived*>(this)->mZOffset, static_cast<Derived*>(this)->mP}; };
43+
44+
constexpr operator S<wrapper::const_reference>() const { return {{}, static_cast<const Derived*>(this)->mX, static_cast<const Derived*>(this)->mC, static_cast<const Derived*>(this)->mZOffset, static_cast<const Derived*>(this)->mP}; };
45+
constexpr operator S<wrapper::const_reference_restrict>() const { return {{}, static_cast<const Derived*>(this)->mX, static_cast<const Derived*>(this)->mC, static_cast<const Derived*>(this)->mZOffset, static_cast<const Derived*>(this)->mP}; };
46+
};
47+
48+
template <template <template <class> class> class S>
49+
struct BaseCRTP<S, wrapper::reference_restrict> {
50+
using Derived = S<wrapper::reference_restrict>;
51+
52+
constexpr operator S<wrapper::reference>() { return {{}, static_cast<Derived*>(this)->mX, static_cast<Derived*>(this)->mC, static_cast<Derived*>(this)->mZOffset, static_cast<Derived*>(this)->mP}; };
53+
54+
constexpr operator S<wrapper::const_reference>() const { return {{}, static_cast<const Derived*>(this)->mX, static_cast<const Derived*>(this)->mC, static_cast<const Derived*>(this)->mZOffset, static_cast<const Derived*>(this)->mP}; };
55+
constexpr operator S<wrapper::const_reference_restrict>() const { return {{}, static_cast<const Derived*>(this)->mX, static_cast<const Derived*>(this)->mC, static_cast<const Derived*>(this)->mZOffset, static_cast<const Derived*>(this)->mP}; };
56+
};
57+
58+
template <template <template <class> class> class S>
59+
struct BaseCRTP<S, wrapper::const_reference> {
60+
using Derived = S<wrapper::const_reference>;
61+
62+
constexpr operator S<wrapper::const_reference_restrict>() const { return {{}, static_cast<const Derived*>(this)->mX, static_cast<const Derived*>(this)->mC, static_cast<const Derived*>(this)->mZOffset, static_cast<const Derived*>(this)->mP}; };
63+
};
64+
65+
template <template <template <class> class> class S>
66+
struct BaseCRTP<S, wrapper::const_reference_restrict> {
67+
using Derived = S<wrapper::const_reference_restrict>;
68+
3769
constexpr operator S<wrapper::const_reference>() const { return {{}, static_cast<const Derived*>(this)->mX, static_cast<const Derived*>(this)->mC, static_cast<const Derived*>(this)->mZOffset, static_cast<const Derived*>(this)->mP}; };
3870
};
3971

‎GPU/GPUTracking/SectorTracker/GPUTPCDef.h‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,12 +32,20 @@ typedef uint32_t cahit;
3232
#endif
3333
struct cahit2 { cahit x, y; };
3434

35-
namespace wrapper {
35+
namespace wrapper
36+
{
3637
template <class T> using value = T;
38+
3739
template <class T> using reference = T&;
3840
template <class T> using const_reference = const T&;
41+
template <class T> using reference_restrict = T& GPUrestrict();
42+
template <class T> using const_reference_restrict = const T& GPUrestrict();
43+
3944
template <class T> using pointer = T*;
40-
} // namespace o2::gpu::wrapper
45+
template <class T> using const_pointer = const T*;
46+
template <class T> using pointer_restrict = T* GPUrestrict();
47+
template <class T> using const_pointer_restrict = const T* GPUrestrict();
48+
} // namespace o2::gpu::wrapper
4149

4250
} // namespace o2::GPU
4351

‎GPU/GPUTracking/SectorTracker/GPUTPCTrackParam.cxx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -879,5 +879,6 @@ GPUd() int32_t GPUTPCTrackParamSkeleton<F>::GetPropagatedYZ(float bz, float x, f
879879
namespace o2::gpu {
880880
template class GPUTPCTrackParamSkeleton<wrapper::value>;
881881
template class GPUTPCTrackParamSkeleton<wrapper::reference>;
882+
template class GPUTPCTrackParamSkeleton<wrapper::reference_restrict>;
882883
//template class GPUTPCTrackParamSkeleton<wrapper::const_reference>;
883884
} // namespace o2::gpu

‎GPU/GPUTracking/SectorTracker/GPUTPCTrackParam.h‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,13 +29,45 @@ struct CRTP { };
2929
template <template <template <class> class> class S>
3030
struct CRTP<S, wrapper::value> {
3131
using Derived = S<wrapper::value>;
32+
3233
constexpr operator S<wrapper::reference>() { return {{}, static_cast<Derived*>(this)->mParam, static_cast<Derived*>(this)->mSignCosPhi, static_cast<Derived*>(this)->mChi2, static_cast<Derived*>(this)->mNDF}; };
34+
constexpr operator S<wrapper::reference_restrict>() { return {{}, static_cast<Derived*>(this)->mParam, static_cast<Derived*>(this)->mSignCosPhi, static_cast<Derived*>(this)->mChi2, static_cast<Derived*>(this)->mNDF}; };
35+
3336
constexpr operator S<wrapper::const_reference>() const { return {{}, static_cast<const Derived*>(this)->mParam, static_cast<const Derived*>(this)->mSignCosPhi, static_cast<const Derived*>(this)->mChi2, static_cast<const Derived*>(this)->mNDF}; };
37+
constexpr operator S<wrapper::const_reference_restrict>() const { return {{}, static_cast<const Derived*>(this)->mParam, static_cast<const Derived*>(this)->mSignCosPhi, static_cast<const Derived*>(this)->mChi2, static_cast<const Derived*>(this)->mNDF}; };
3438
};
3539

3640
template <template <template <class> class> class S>
3741
struct CRTP<S, wrapper::reference> {
3842
using Derived = S<wrapper::reference>;
43+
44+
constexpr operator S<wrapper::reference_restrict>() { return {{}, static_cast<Derived*>(this)->mParam, static_cast<Derived*>(this)->mSignCosPhi, static_cast<Derived*>(this)->mChi2, static_cast<Derived*>(this)->mNDF}; };
45+
constexpr operator S<wrapper::const_reference>() const { return {{}, static_cast<const Derived*>(this)->mParam, static_cast<const Derived*>(this)->mSignCosPhi, static_cast<const Derived*>(this)->mChi2, static_cast<const Derived*>(this)->mNDF}; };
46+
47+
constexpr operator S<wrapper::const_reference_restrict>() const { return {{}, static_cast<const Derived*>(this)->mParam, static_cast<const Derived*>(this)->mSignCosPhi, static_cast<const Derived*>(this)->mChi2, static_cast<const Derived*>(this)->mNDF}; };
48+
};
49+
50+
template <template <template <class> class> class S>
51+
struct CRTP<S, wrapper::reference_restrict> {
52+
using Derived = S<wrapper::reference_restrict>;
53+
54+
constexpr operator S<wrapper::reference>() { return {{}, static_cast<Derived*>(this)->mParam, static_cast<Derived*>(this)->mSignCosPhi, static_cast<Derived*>(this)->mChi2, static_cast<Derived*>(this)->mNDF}; };
55+
56+
constexpr operator S<wrapper::const_reference>() const { return {{}, static_cast<const Derived*>(this)->mParam, static_cast<const Derived*>(this)->mSignCosPhi, static_cast<const Derived*>(this)->mChi2, static_cast<const Derived*>(this)->mNDF}; };
57+
constexpr operator S<wrapper::const_reference_restrict>() const { return {{}, static_cast<const Derived*>(this)->mParam, static_cast<const Derived*>(this)->mSignCosPhi, static_cast<const Derived*>(this)->mChi2, static_cast<const Derived*>(this)->mNDF}; };
58+
};
59+
60+
template <template <template <class> class> class S>
61+
struct CRTP<S, wrapper::const_reference> {
62+
using Derived = S<wrapper::const_reference>;
63+
64+
constexpr operator S<wrapper::const_reference_restrict>() const { return {{}, static_cast<const Derived*>(this)->mParam, static_cast<const Derived*>(this)->mSignCosPhi, static_cast<const Derived*>(this)->mChi2, static_cast<const Derived*>(this)->mNDF}; };
65+
};
66+
67+
template <template <template <class> class> class S>
68+
struct CRTP<S, wrapper::const_reference_restrict> {
69+
using Derived = S<wrapper::const_reference_restrict>;
70+
3971
constexpr operator S<wrapper::const_reference>() const { return {{}, static_cast<const Derived*>(this)->mParam, static_cast<const Derived*>(this)->mSignCosPhi, static_cast<const Derived*>(this)->mChi2, static_cast<const Derived*>(this)->mNDF}; };
4072
};
4173

‎GPU/GPUTracking/SectorTracker/GPUTPCTracker.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -186,7 +186,7 @@ class GPUTPCTracker : public GPUProcessor
186186
GPUhd() GPUglobalref() GPUTPCHitId* TrackletStartHits() { return mTrackletStartHits; }
187187
GPUhd() GPUglobalref() GPUTPCHitId* TrackletTmpStartHits() const { return mTrackletTmpStartHits; }
188188

189-
GPUhd() GPUglobalref() GPUTPCTracklet_reference Tracklet(int32_t i) {
189+
GPUhd() GPUglobalref() GPUTPCTrackletSkeleton<wrapper::reference_restrict> Tracklet(int32_t i) {
190190
return {
191191
mTracklets.mFirstRow[i],
192192
mTracklets.mLastRow[i],

‎GPU/GPUTracking/SectorTracker/GPUTPCTracklet.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,6 @@ class GPUTPCTrackletSkeleton
7979
};
8080

8181
using GPUTPCTracklet = GPUTPCTrackletSkeleton<wrapper::value>;
82-
using GPUTPCTracklet_reference = GPUTPCTrackletSkeleton<wrapper::reference>;
8382

8483
} // namespace o2::gpu
8584

‎GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.cxx‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,15 +29,13 @@
2929

3030
using namespace o2::gpu;
3131

32-
using GPUTPCTrackParam_reference = GPUTPCTrackParamSkeleton<wrapper::reference>;
33-
34-
GPUdii() void GPUTPCTrackletConstructor::InitTracklet(GPUTPCTrackParam_reference GPUrestrict() tParam)
32+
GPUdii() void GPUTPCTrackletConstructor::InitTracklet(GPUTPCTrackParamSkeleton<wrapper::reference_restrict> tParam)
3533
{
3634
// Initialize Tracklet Parameters using default values
3735
tParam.InitParam();
3836
}
3937

40-
GPUd() bool GPUTPCTrackletConstructor::CheckCov(GPUTPCTrackParam_reference GPUrestrict() tParam)
38+
GPUd() bool GPUTPCTrackletConstructor::CheckCov(GPUTPCTrackParamSkeleton<wrapper::reference_restrict> tParam)
4139
{
4240
bool ok = 1;
4341
const float* c = tParam.Cov();
@@ -54,7 +52,7 @@ GPUd() bool GPUTPCTrackletConstructor::CheckCov(GPUTPCTrackParam_reference GPUre
5452
return (ok);
5553
}
5654

57-
GPUd() void GPUTPCTrackletConstructor::StoreTracklet(int32_t /*nBlocks*/, int32_t /*nThreads*/, int32_t /*iBlock*/, int32_t /*iThread*/, GPUsharedref() GPUSharedMemory& s, GPUTPCThreadMemory& GPUrestrict() r, GPUconstantref() GPUTPCTracker& GPUrestrict() tracker, GPUTPCTrackParam_reference GPUrestrict() tParam, calink* rowHits)
55+
GPUd() void GPUTPCTrackletConstructor::StoreTracklet(int32_t /*nBlocks*/, int32_t /*nThreads*/, int32_t /*iBlock*/, int32_t /*iThread*/, GPUsharedref() GPUSharedMemory& s, GPUTPCThreadMemory& GPUrestrict() r, GPUconstantref() GPUTPCTracker& GPUrestrict() tracker, GPUTPCTrackParamSkeleton<wrapper::reference_restrict> tParam, calink* rowHits)
5856
{
5957
// reconstruction of tracklets, tracklet store step
6058
const uint32_t nHits = r.mLastRow + 1 - r.mFirstRow;
@@ -80,7 +78,7 @@ GPUd() void GPUTPCTrackletConstructor::StoreTracklet(int32_t /*nBlocks*/, int32_
8078
return;
8179
}
8280

83-
GPUglobalref() GPUTPCTracklet_reference GPUrestrict() tracklet = tracker.Tracklet(itrout);
81+
GPUglobalref() GPUTPCTrackletSkeleton<wrapper::reference_restrict> tracklet = tracker.Tracklet(itrout);
8482

8583
CADEBUG(printf(" Storing tracklet: %d rows\n", nHits));
8684

@@ -105,7 +103,7 @@ GPUd() void GPUTPCTrackletConstructor::StoreTracklet(int32_t /*nBlocks*/, int32_
105103
}
106104

107105
template <class T>
108-
GPUdic(2, 1) void GPUTPCTrackletConstructor::UpdateTracklet(int32_t /*nBlocks*/, int32_t /*nThreads*/, int32_t /*iBlock*/, int32_t /*iThread*/, GPUsharedref() T& s, GPUTPCThreadMemory& GPUrestrict() r, GPUconstantref() GPUTPCTracker& GPUrestrict() tracker, GPUTPCTrackParam_reference GPUrestrict() tParam, int32_t iRow, calink& rowHit, calink* rowHits)
106+
GPUdic(2, 1) void GPUTPCTrackletConstructor::UpdateTracklet(int32_t /*nBlocks*/, int32_t /*nThreads*/, int32_t /*iBlock*/, int32_t /*iThread*/, GPUsharedref() T& s, GPUTPCThreadMemory& GPUrestrict() r, GPUconstantref() GPUTPCTracker& GPUrestrict() tracker, GPUTPCTrackParamSkeleton<wrapper::reference_restrict> tParam, int32_t iRow, calink& rowHit, calink* rowHits)
109107
{
110108
// reconstruction of tracklets, tracklets update step
111109
CA_MAKE_SHARED_REF(GPUTPCRow, row, tracker.Row(iRow), s.mRows[iRow]);
@@ -492,7 +490,7 @@ GPUdii() void GPUTPCTrackletConstructor::Thread(int32_t nBlocks, int32_t nThread
492490
}
493491

494492
template <> // FIXME: GPUgeneric() needed to make the clang spirv output link correctly
495-
GPUd() int32_t GPUTPCTrackletConstructor::GPUTPCTrackletConstructorExtrapolationTracking<GPUgeneric() GPUTPCExtrapolationTracking::GPUSharedMemory>(GPUconstantref() GPUTPCTracker& GPUrestrict() tracker, GPUsharedref() GPUTPCExtrapolationTracking::GPUSharedMemory& sMem, GPUTPCTrackParam_reference GPUrestrict() tParam, int32_t row, int32_t increment, int32_t iTracklet, calink* rowHits)
493+
GPUd() int32_t GPUTPCTrackletConstructor::GPUTPCTrackletConstructorExtrapolationTracking<GPUgeneric() GPUTPCExtrapolationTracking::GPUSharedMemory>(GPUconstantref() GPUTPCTracker& GPUrestrict() tracker, GPUsharedref() GPUTPCExtrapolationTracking::GPUSharedMemory& sMem, GPUTPCTrackParamSkeleton<wrapper::reference_restrict> tParam, int32_t row, int32_t increment, int32_t iTracklet, calink* rowHits)
496494
{
497495
GPUTPCThreadMemory rMem;
498496
rMem.mISH = iTracklet;

‎GPU/GPUTracking/SectorTracker/GPUTPCTrackletConstructor.h‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -70,19 +70,19 @@ class GPUTPCTrackletConstructor : public GPUKernelTemplate
7070
#endif // GPUCA_TRACKLET_CONSTRUCTOR_DO_PROFILE
7171
};
7272

73-
GPUd() static void InitTracklet(GPUTPCTrackParamSkeleton<wrapper::reference> tParam);
73+
GPUd() static void InitTracklet(GPUTPCTrackParamSkeleton<wrapper::reference_restrict> tParam);
7474

7575
template <class T>
76-
GPUd() static void UpdateTracklet(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() T& s, GPUTPCThreadMemory& r, GPUconstantref() GPUTPCTracker& tracker, GPUTPCTrackParamSkeleton<wrapper::reference> tParam, int32_t iRow, calink& rowHit, calink* rowHits);
76+
GPUd() static void UpdateTracklet(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() T& s, GPUTPCThreadMemory& r, GPUconstantref() GPUTPCTracker& tracker, GPUTPCTrackParamSkeleton<wrapper::reference_restrict> tParam, int32_t iRow, calink& rowHit, calink* rowHits);
7777

78-
GPUd() static void StoreTracklet(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& s, GPUTPCThreadMemory& r, GPUconstantref() GPUTPCTracker& tracker, GPUTPCTrackParamSkeleton<wrapper::reference> tParam, calink* rowHits);
78+
GPUd() static void StoreTracklet(int32_t nBlocks, int32_t nThreads, int32_t iBlock, int32_t iThread, GPUsharedref() GPUSharedMemory& s, GPUTPCThreadMemory& r, GPUconstantref() GPUTPCTracker& tracker, GPUTPCTrackParamSkeleton<wrapper::reference_restrict> tParam, calink* rowHits);
7979

80-
GPUd() static bool CheckCov(GPUTPCTrackParamSkeleton<wrapper::reference> tParam);
80+
GPUd() static bool CheckCov(GPUTPCTrackParamSkeleton<wrapper::reference_restrict> tParam);
8181

8282
GPUd() static void DoTracklet(GPUconstantref() GPUTPCTracker& tracker, GPUsharedref() GPUTPCTrackletConstructor::GPUSharedMemory& sMem, GPUTPCThreadMemory& rMem);
8383

8484
template <class T>
85-
GPUd() static int32_t GPUTPCTrackletConstructorExtrapolationTracking(GPUconstantref() GPUTPCTracker& tracker, GPUsharedref() T& sMem, GPUTPCTrackParamSkeleton<wrapper::reference> tParam, int32_t startrow, int32_t increment, int32_t iTracklet, calink* rowHits);
85+
GPUd() static int32_t GPUTPCTrackletConstructorExtrapolationTracking(GPUconstantref() GPUTPCTracker& tracker, GPUsharedref() T& sMem, GPUTPCTrackParamSkeleton<wrapper::reference_restrict> tParam, int32_t startrow, int32_t increment, int32_t iTracklet, calink* rowHits);
8686

8787
typedef GPUconstantref() GPUTPCTracker processorType;
8888
GPUhdi() constexpr static GPUDataTypes::RecoStep GetRecoStep() { return GPUDataTypes::RecoStep::TPCSectorTracking; }

‎GPU/GPUTracking/SectorTracker/GPUTPCTrackletSelector.cxx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ GPUdii() void GPUTPCTrackletSelector::Thread<0>(int32_t nBlocks, int32_t nThread
3939
for (int32_t itr = s.mItr0 + iThread; itr < s.mNTracklets; itr += s.mNThreadsTotal) {
4040
GPUbarrierWarp();
4141

42-
GPUglobalref() GPUTPCTracklet_reference GPUrestrict() tracklet = tracker.Tracklet(itr);
42+
GPUglobalref() GPUTPCTrackletSkeleton<wrapper::reference_restrict> tracklet = tracker.Tracklet(itr);
4343

4444
int32_t firstRow = tracklet.FirstRow();
4545
int32_t lastRow = tracklet.LastRow();

0 commit comments

Comments
 (0)