ITS: slab allocator idea

Detectors/ITSMFT/ITS/tracking/CMakeLists.txt

@@ -13,6 +13,7 @@ o2_add_library(ITStracking
                TARGETVARNAME targetName
                SOURCES src/ClusterLines.cxx
                        src/Cluster.cxx
+                       src/CapacityEstimator.cxx
                        src/Configuration.cxx
                        src/FastMultEstConfig.cxx
                        src/FastMultEst.cxx

Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/LaunchGeometry.h

@@ -0,0 +1,68 @@
+// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+///
+/// \file LaunchGeometry.h
+/// \brief Compile-time launch geometry of the ITS tracking kernels, per GPU family.
+///
+
+#ifndef ITSTRACKINGGPU_LAUNCHGEOMETRY_H_
+#define ITSTRACKINGGPU_LAUNCHGEOMETRY_H_
+
+namespace o2::its::gpu
+{
+
+#if defined(GPUCA_GPUTYPE_VEGA) // gfx906: MI50, Radeon VII
+constexpr int ComputeUnits = 60;
+constexpr int WarpSize = 64;
+#elif defined(GPUCA_GPUTYPE_MI100)     // gfx908
+constexpr int ComputeUnits = 120;
+constexpr int WarpSize = 64;
+#elif defined(GPUCA_GPUTYPE_MI210)     // gfx90a
+constexpr int ComputeUnits = 104;
+constexpr int WarpSize = 64;
+#elif defined(GPUCA_GPUTYPE_MI300)     // gfx942: MI300X (MI300A has 228)
+constexpr int ComputeUnits = 304;
+constexpr int WarpSize = 64;
+#elif defined(GPUCA_GPUTYPE_RDNA)      // gfx10xx/11xx consumer parts, wave32
+constexpr int ComputeUnits = 60;
+constexpr int WarpSize = 32;
+#elif defined(GPUCA_GPUTYPE_BLACKWELL) // sm_120: RTX 5080
+constexpr int ComputeUnits = 84;
+constexpr int WarpSize = 32;
+#elif defined(GPUCA_GPUTYPE_HOPPER)    // sm_90: H100
+constexpr int ComputeUnits = 132;
+constexpr int WarpSize = 32;
+#elif defined(GPUCA_GPUTYPE_ADA)       // sm_89: RTX 4090
+constexpr int ComputeUnits = 128;
+constexpr int WarpSize = 32;
+#elif defined(GPUCA_GPUTYPE_AMPERE)    // sm_80/86: A100 has 108, RTX 3090 has 82
+constexpr int ComputeUnits = 108;
+constexpr int WarpSize = 32;
+#elif defined(GPUCA_GPUTYPE_TURING)    // sm_75: RTX 2080 Ti
+constexpr int ComputeUnits = 68;
+constexpr int WarpSize = 32;
+#else
+#warning "GPU architecture not available setting fallback"
+constexpr int ComputeUnits = 60;
+constexpr int WarpSize = 64;
+#endif
+
+constexpr int GPUThreads = 256;
+constexpr int BlocksPerComputeUnit = 4;
+constexpr int GPUBlocks = ComputeUnits * BlocksPerComputeUnit;
+constexpr int GPUThreadsTotal = GPUBlocks * GPUThreads;
+
+static_assert(GPUThreads % WarpSize == 0, "block size must be a whole number of warps/waves");
+static_assert(GPUBlocks > 0 && GPUThreads > 0, "degenerate launch geometry");
+
+} // namespace o2::its::gpu
+
+#endif // ITSTRACKINGGPU_LAUNCHGEOMETRY_H_

Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TimeFrameGPU.h

@@ -45,68 +45,55 @@ class TimeFrameGPU : public TimeFrame<NLayers>
   /// Most relevant operations
   void pushMemoryStack(const int);
   void popMemoryStack(const int);
-  void registerHostMemory(const int);
-  void unregisterHostMemory(const int);
+  void unregisterHostMemory();
   void initialise(const TrackingParameters&, int maxLayers);
   void initialise(const TrackingParameters&, int maxLayers, int iteration);
   void loadIndexTableUtils();
   void loadTrackingTopologies();
   void loadTrackingFrameInfoDevice(const int);
-  void createTrackingFrameInfoDeviceArray();
+  void createTrackingFrameInfoDeviceArray(const int = NLayers);
   void loadUnsortedClustersDevice(const int);
   void createUnsortedClustersDeviceArray(const int = NLayers);
   void loadClustersDevice(const int);
   void createClustersDeviceArray(const int = NLayers);
   void loadClustersIndexTables(const int);
-  void createClustersIndexTablesArray();
+  void createClustersIndexTablesArray(const int = NLayers);
   void createUsedClustersDevice(const int);
   void createUsedClustersDeviceArray(const int = NLayers);
   void loadUsedClustersDevice();
   void loadROFrameClustersDevice(const int);
-  void createROFrameClustersDeviceArray();
+  void createROFrameClustersDeviceArray(const int = NLayers);
   void loadROFCutMask(const int);
   void loadVertices();
   void loadROFOverlapTable();
   void loadROFVertexLookupTable();
-  void updateROFVertexLookupTable();
+  void uploadROFVertexLookupTable();
+  void loadIterationParameters(const TrackingParameters&);
 
   ///
   void createTrackletsLUTDevice(bool, const int);
   void createTrackletsLUTDeviceArray();
-  void loadTrackletsDevice();
-  void loadTrackletsLUTDevice();
-  void loadCellsDevice();
-  void loadCellsLUTDevice();
-  void loadTrackSeedsDevice();
-  void loadTrackSeedsChi2Device();
-  void loadTrackSeedsDevice(bounded_vector<TrackSeedN>&);
-  void createTrackletsBuffers(const int);
+  void createTrackSeedsDevice(const size_t capacity);
+  void createTrackletsBuffers(const int, size_t capacity);
   void createTrackletsBuffersArray();
-  void createCellsBuffers(const int);
+  void createCellsBuffers(const int, size_t capacity);
   void createCellsBuffersArray();
-  void createCellsDevice();
   void createCellsLUTDevice(const int);
   void createCellsLUTDeviceArray();
-  void createNeighboursIndexTablesDevice(const int);
-  void createNeighboursDevice(const unsigned int layer);
+  void createNeighboursDevice(const unsigned int layer, size_t capacity);
   void createNeighboursLUTDevice(const int, const unsigned int);
-  void createTrackITSExtDevice(const size_t);
+  void createTrackITSExtDevice(const size_t capacity);
+  void createTrackITSExtHost(const size_t nTracks);
   void createTrackExtensionScratchDevice(const int nThreads, const int maxHypotheses);
   void downloadTrackITSExtDevice();
   void downloadTrackIndicesDevice();
-  void downloadCellsNeighboursDevice(std::vector<bounded_vector<CellNeighbour>>&, const int);
-  void downloadNeighboursLUTDevice(bounded_vector<int>&, const int);
-  void downloadCellsDevice();
-  void downloadCellsLUTDevice();
 
   /// synchronization
   auto& getStream(const size_t stream) { return mGpuStreams[stream]; }
   auto& getStreams() { return mGpuStreams; }
-  void syncStream(const size_t stream);
   void syncStreams(const bool = true);
   void waitEvent(const int, const int);
   void recordEvent(const int);
-  void recordEvents(const int = 0, const int = NLayers);
 
   /// cleanup
   virtual void wipe() final;
@@ -115,29 +102,25 @@ class TimeFrameGPU : public TimeFrame<NLayers>
   virtual bool isGPU() const noexcept final { return true; }
   virtual const char* getName() const noexcept override final { return "GPU"; }
   IndexTableUtilsN* getDeviceIndexTableUtils() { return mIndexTableUtilsDevice; }
+  const float* getDeviceLayerRadii() const { return mLayerRadiiDevice; }
+  const float* getDeviceMinPts() const { return mMinPtsDevice; }
+  const float* getDeviceLayerxX0() const { return mLayerxX0Device; }
   const auto getDeviceROFOverlapTableView() { return mDeviceROFOverlapTableView; }
   const auto getDeviceROFVertexLookupTableView() { return mDeviceROFVertexLookupTableView; }
   const auto getDeviceROFMaskTableView() { return mDeviceROFMaskTableView; }
   const auto getDeviceTrackingTopologyView() const { return mDeviceTrackingTopologyView; }
-  int* getDeviceROFramesClusters(const int layer) { return mROFramesClustersDevice[layer]; }
   auto& getTrackITSExt() { return mTrackITSExt; }
   auto& getTrackIndices() { return mTrackIndices; }
   Vertex* getDeviceVertices() { return mPrimaryVerticesDevice; }
-  int* getDeviceROFramesPV() { return mROFramesPVDevice; }
-  unsigned char* getDeviceUsedClusters(const int);
-  const o2::base::Propagator* getChainPropagator();
 
   // Hybrid
   TrackITSExt* getDeviceTrackITSExt() { return mTrackITSExtDevice; }
   int* getDeviceTrackIndices() { return mTrackIndicesDevice; }
   TrackExtensionHypothesis<NLayers>* getDeviceActiveTrackExtensionHypotheses() { return mActiveTrackExtensionHypothesesDevice; }
   TrackExtensionHypothesis<NLayers>* getDeviceNextTrackExtensionHypotheses() { return mNextTrackExtensionHypothesesDevice; }
   int* getDeviceNeighboursLUT(const int layer) { return mNeighboursLUTDevice[layer]; }
-  gsl::span<int*> getDeviceNeighboursLUTs() { return mNeighboursLUTDevice; }
   CellNeighbour** getDeviceArrayNeighbours() { return mNeighboursDeviceArray; }
-  std::array<CellNeighbour*, MaxCells>& getDeviceNeighboursAll() { return mNeighboursDevice; }
   CellNeighbour* getDeviceNeighbours(const int layer) { return mNeighboursDevice[layer]; }
-  TrackingFrameInfo* getDeviceTrackingFrameInfo(const int);
   const TrackingFrameInfo** getDeviceArrayTrackingFrameInfo() const { return mTrackingFrameInfoDeviceArray; }
   const Cluster** getDeviceArrayClusters() const { return mClustersDeviceArray; }
   const Cluster** getDeviceArrayUnsortedClusters() const { return mUnsortedClustersDeviceArray; }
@@ -151,11 +134,9 @@ class TimeFrameGPU : public TimeFrame<NLayers>
   int** getDeviceArrayNeighboursCellLUT() const { return mNeighboursCellLUTDeviceArray; }
   CellSeed** getDeviceArrayCells() { return mCellsDeviceArray; }
   TrackSeedN* getDeviceTrackSeeds() { return mTrackSeedsDevice; }
-  int* getDeviceTrackSeedsLUT() { return mTrackSeedsLUTDevice; }
+  int* getDeviceTrackSeedIndices() { return mTrackSeedIndicesDevice; }
+  int* getDeviceTrackCounter() { return mTrackCounterDevice; }
   auto getNTrackSeeds() const { return mNTracks; }
-  o2::track::TrackParCovF** getDeviceArrayTrackSeeds() { return mCellSeedsDeviceArray; }
-  float** getDeviceArrayTrackSeedsChi2() { return mCellSeedsChi2DeviceArray; }
-  int* getDeviceNeighboursIndexTables(const int layer) { return mNeighboursIndexTablesDevice[layer]; }
 
   void setDevicePropagator(const o2::base::PropagatorImpl<float>* p) final { this->mPropagatorDevice = p; }
 
@@ -164,7 +145,6 @@ class TimeFrameGPU : public TimeFrame<NLayers>
   gsl::span<int> getNCells() { return {mNCells.data(), static_cast<gsl::span<int>::size_type>(this->mTrackingTopologyView.nCells)}; }
   auto& getArrayNCells() { return mNCells; }
   gsl::span<int> getNNeighbours() { return {mNNeighbours.data(), static_cast<gsl::span<int>::size_type>(this->mTrackingTopologyView.nCells)}; }
-  auto& getArrayNNeighbours() { return mNNeighbours; }
 
   // Host-available device getters
   gsl::span<int*> getDeviceTrackletsLUTs() { return mTrackletsLUTDevice; }
@@ -178,16 +158,45 @@ class TimeFrameGPU : public TimeFrame<NLayers>
   size_t getNumberOfNeighbours() const final;
 
  private:
-  void allocMemAsync(void**, size_t, Stream&, bool, int32_t = o2::gpu::GPUMemoryResource::MEMORY_GPU); // Abstract owned and unowned memory allocations on specific stream
-  void allocMem(void**, size_t, bool, int32_t = o2::gpu::GPUMemoryResource::MEMORY_GPU);               // Abstract owned and unowned memory allocations on default stream
+  enum class SlotInit {
+    Raw, ///< whatever the allocator handed back
+    Zero ///< cleared on the slot's stream
+  };
+
+  template <typename T>
+  T* allocDevice(size_t n, int32_t type = o2::gpu::GPUMemoryResource::MEMORY_GPU);
+  template <typename T>
+  T* allocDeviceAsync(size_t n, Stream&, int32_t type = o2::gpu::GPUMemoryResource::MEMORY_GPU);
+  template <typename SlotPtr>
+  SlotPtr* allocSlotArray(size_t n);
+  template <typename T>
+  void copyToDevice(T* dst, const T* src, size_t n);
+  template <typename T>
+  void copyFromDevice(T* dst, const T* src, size_t n);
+  template <typename T, typename ArrayT>
+  void publishSlot(ArrayT deviceArray, int slot, T* const& devicePtr, Stream&);
+  template <typename T, size_t N, typename ArrayT>
+  T* createSlot(std::array<T*, N>& slots, ArrayT deviceArray, int slot, size_t n, const char* what, SlotInit init = SlotInit::Raw, int32_t type = o2::gpu::GPUMemoryResource::MEMORY_GPU);
+  template <typename T, size_t N, typename ArrayT, typename Container>
+  void uploadSlot(std::array<T*, N>& slots, ArrayT deviceArray, int slot, const Container& host, const char* what);
+  template <typename ArrayT, typename T, size_t N>
+  void createPinnedSlotArray(ArrayT& deviceArray, std::array<T*, N>& slots, std::bitset<NLayers + 1>& pinned);
+  template <typename Layers>
+  void pinHostLayers(Layers& layers, std::bitset<NLayers + 1>& pinned, int maxLayers);
+  template <typename Table>
+  typename Table::View uploadNavigationTable(const Table& table, const typename Table::View& hostView);
 
   // Host-available device buffer sizes
   std::array<int, MaxLinks> mNTracklets{};
   std::array<int, MaxCells> mNCells{};
   std::array<int, MaxCells> mNNeighbours{};
 
   // Device pointers
-  IndexTableUtilsN* mIndexTableUtilsDevice;
+  IndexTableUtilsN* mIndexTableUtilsDevice{nullptr};
+  float* mIterationParametersDevice{nullptr};
+  const float* mLayerRadiiDevice{nullptr};
+  const float* mMinPtsDevice{nullptr};
+  const float* mLayerxX0Device{nullptr};
   // device navigation views
   ROFOverlapTableN::View mDeviceROFOverlapTableView;
   ROFVertexLookupTableN::View mDeviceROFVertexLookupTableView;
@@ -196,18 +205,17 @@ class TimeFrameGPU : public TimeFrame<NLayers>
   typename TrackingTopologyN::View mDeviceTrackingTopologyView;
 
   // Hybrid pref
-  Vertex* mPrimaryVerticesDevice;
-  int* mROFramesPVDevice;
-  std::array<Cluster*, NLayers> mClustersDevice;
-  std::array<Cluster*, NLayers> mUnsortedClustersDevice;
-  std::array<int*, NLayers> mClustersIndexTablesDevice;
-  std::array<unsigned char*, NLayers> mUsedClustersDevice;
-  std::array<int*, NLayers> mROFramesClustersDevice;
-  const Cluster** mClustersDeviceArray;
-  const Cluster** mUnsortedClustersDeviceArray;
-  const int** mClustersIndexTablesDeviceArray;
-  uint8_t** mUsedClustersDeviceArray;
-  const int** mROFramesClustersDeviceArray;
+  Vertex* mPrimaryVerticesDevice{nullptr};
+  std::array<Cluster*, NLayers> mClustersDevice{};
+  std::array<Cluster*, NLayers> mUnsortedClustersDevice{};
+  std::array<int*, NLayers> mClustersIndexTablesDevice{};
+  std::array<unsigned char*, NLayers> mUsedClustersDevice{};
+  std::array<int*, NLayers> mROFramesClustersDevice{};
+  const Cluster** mClustersDeviceArray{nullptr};
+  const Cluster** mUnsortedClustersDeviceArray{nullptr};
+  const int** mClustersIndexTablesDeviceArray{nullptr};
+  uint8_t** mUsedClustersDeviceArray{nullptr};
+  const int** mROFramesClustersDeviceArray{nullptr};
   std::array<Tracklet*, MaxLinks> mTrackletsDevice{};
   std::array<int*, MaxLinks> mTrackletsLUTDevice{};
   std::array<int*, MaxCells> mCellsLUTDevice{};
@@ -218,24 +226,20 @@ class TimeFrameGPU : public TimeFrame<NLayers>
   int** mNeighboursCellLUTDeviceArray{nullptr};
   int** mTrackletsLUTDeviceArray{nullptr};
   std::array<CellSeed*, MaxCells> mCellsDevice{};
-  CellSeed** mCellsDeviceArray;
-  std::array<int*, MaxCells> mNeighboursIndexTablesDevice{};
+  CellSeed** mCellsDeviceArray{nullptr};
   TrackSeedN* mTrackSeedsDevice{nullptr};
-  int* mTrackSeedsLUTDevice{nullptr};
+  int* mTrackSeedIndicesDevice{nullptr}; ///< which seed each emitted track was fitted from
+  int* mTrackCounterDevice{nullptr};
   unsigned int mNTracks{0};
-  std::array<o2::track::TrackParCovF*, MaxCells> mCellSeedsDevice{};
-  o2::track::TrackParCovF** mCellSeedsDeviceArray;
-  std::array<float*, MaxCells> mCellSeedsChi2Device{};
-  float** mCellSeedsChi2DeviceArray;
 
-  TrackITSExt* mTrackITSExtDevice;
+  TrackITSExt* mTrackITSExtDevice{nullptr};
   int* mTrackIndicesDevice{nullptr};
   TrackExtensionHypothesis<NLayers>* mActiveTrackExtensionHypothesesDevice{nullptr};
   TrackExtensionHypothesis<NLayers>* mNextTrackExtensionHypothesesDevice{nullptr};
   std::array<CellNeighbour*, MaxCells> mNeighboursDevice{};
   CellNeighbour** mNeighboursDeviceArray{nullptr};
-  std::array<TrackingFrameInfo*, NLayers> mTrackingFrameInfoDevice;
-  const TrackingFrameInfo** mTrackingFrameInfoDeviceArray;
+  std::array<TrackingFrameInfo*, NLayers> mTrackingFrameInfoDevice{};
+  const TrackingFrameInfo** mTrackingFrameInfoDeviceArray{nullptr};
 
   // State
   Streams mGpuStreams;

Detectors/ITSMFT/ITS/tracking/GPU/ITStrackingGPU/TrackerTraitsGPU.h

@@ -22,8 +22,6 @@ namespace o2::its
 template <int NLayers>
 class TrackerTraitsGPU final : public TrackerTraits<NLayers>
 {
-  using typename TrackerTraits<NLayers>::IndexTableUtilsN;
-
  public:
   TrackerTraitsGPU() = default;
   ~TrackerTraitsGPU() final = default;
@@ -47,7 +45,6 @@ class TrackerTraitsGPU final : public TrackerTraits<NLayers>
   int getTFNumberOfCells() const override;
 
  private:
-  IndexTableUtilsN* mDeviceIndexTableUtils;
   gpu::TimeFrameGPU<NLayers>* mTimeFrameGPU;
 };