Add skeleton structure for WritableWarm Directories and IndexInputs

Signed-off-by: Kavya Aggarwal <kavyaagg@amazon.com>

Author: Kavya Aggarwal

CHANGELOG.md

@@ -31,6 +31,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 - Add warmup phase to wait for lag to catch up in pull-based ingestion before serving ([#20526](https://github.com/opensearch-project/OpenSearch/pull/20526))
 - Add a new static method to IndicesOptions API to expose `STRICT_EXPAND_OPEN_HIDDEN_FORBID_CLOSED` index option ([#20980](https://github.com/opensearch-project/OpenSearch/pull/20980))
+- Add skeleton structure for tiered-storage module ([#21017](https://github.com/opensearch-project/OpenSearch/pull/21017))
+- Add skeleton structure for WritableWarm Directories and IndexInputs ([#21082](https://github.com/opensearch-project/OpenSearch/pull/21082))
 
 ### Changed
 - Make telemetry `Tags` immutable ([#20788](https://github.com/opensearch-project/OpenSearch/pull/20788))

server/src/main/java/org/opensearch/storage/directory/OSBlockHotDirectory.java

@@ -0,0 +1,86 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.storage.directory;
+
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FilterDirectory;
+import org.opensearch.index.IndexSettings;
+import org.opensearch.index.store.RemoteSegmentStoreDirectory;
+import org.opensearch.threadpool.ThreadPool;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Supplier;
+
+/**
+ * A directory implementation that stores files as blocks.
+ * This directory manages operations on block files and exposes an interface to read
+ * these block files in non block manner.
+ * If a file is present as is, in a non block manner, this directory can manage such a file as well.
+ * Non block files can get created if the files are created without the provided interfaces.
+ *
+ * FilterDirectory overrides (listAll, fileLength, deleteFile, openInput, copyFrom, etc.),
+ * block file management, and metadata initialization will be added in the implementation PR.
+ */
+public class OSBlockHotDirectory extends FilterDirectory {
+
+    private Logger logger;
+    private final RemoteSegmentStoreDirectory remoteSegmentStoreDirectory;
+    private final IndexSettings indexSettings;
+    private final Path directoryPath;
+    // BlockTransferManager field will be added in the implementation PR
+    private final int blockSizeShift;
+
+    /**
+     * This map stores file length for logical files which are backed by block files.
+     * Writes can happen to this directory outside the block context using createOutput or directly
+     * on the FSDirectory path. Those file names will not be present in this map and will not be
+     * treated as block files.
+     */
+    protected final Map<String, Long> logicalFileLengthMap = new ConcurrentHashMap<>();
+
+    /**
+     * Creates a new OSBlockHotDirectory instance with default block transfer manager.
+     *
+     * @param delegate The underlying directory to delegate operations to
+     * @param remoteDirectory The remote directory for segment files
+     * @param indexSettings Index-specific settings
+     * @param threadPoolSupplier Supplier for the thread pool used for async block downloads
+     * @throws IllegalArgumentException if no FSDirectory is found in delegate chain
+     */
+    public OSBlockHotDirectory(
+        Directory delegate,
+        Directory remoteDirectory,
+        IndexSettings indexSettings,
+        Supplier<ThreadPool> threadPoolSupplier
+    ) throws IOException {
+        super(delegate);
+        throw new UnsupportedOperationException("Not yet implemented");
+    }
+
+    /**
+     * Creates a new OSBlockHotDirectory instance with a custom block transfer manager.
+     *
+     * @param delegate The underlying directory to delegate operations to
+     * @param remoteDirectory The remote directory for storing segments
+     * @param indexSettings Index-specific settings
+     * @throws IllegalArgumentException if validation fails
+     */
+    public OSBlockHotDirectory(
+        Directory delegate,
+        Directory remoteDirectory,
+        IndexSettings indexSettings
+    ) throws IOException {
+        super(delegate);
+        throw new UnsupportedOperationException("Not yet implemented");
+    }
+}

server/src/main/java/org/opensearch/storage/directory/OSBlockHotDirectoryFactory.java

@@ -0,0 +1,51 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.storage.directory;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.store.Directory;
+import org.opensearch.index.IndexSettings;
+import org.opensearch.index.shard.ShardPath;
+import org.opensearch.index.store.remote.filecache.FileCache;
+import org.opensearch.plugins.IndexStorePlugin;
+import org.opensearch.threadpool.ThreadPool;
+
+import java.io.IOException;
+import java.util.function.Supplier;
+
+/**
+ * Factory to create OSBlockHotDirectory.
+ * The newDirectory implementation will be added in the implementation PR.
+ */
+public class OSBlockHotDirectoryFactory implements IndexStorePlugin.CompositeDirectoryFactory {
+
+    private static final Logger logger = LogManager.getLogger(OSBlockHotDirectoryFactory.class);
+    private final Supplier<ThreadPool> threadPoolSupplier;
+
+    /**
+     * Constructs a new OSBlockHotDirectoryFactory.
+     * @param threadPoolSupplier supplier for the thread pool
+     */
+    public OSBlockHotDirectoryFactory(Supplier<ThreadPool> threadPoolSupplier) {
+        this.threadPoolSupplier = threadPoolSupplier;
+    }
+
+    @Override
+    public Directory newDirectory(
+        IndexSettings indexSettings,
+        ShardPath shardPath,
+        IndexStorePlugin.DirectoryFactory directoryFactory,
+        Directory directory,
+        FileCache fileCache,
+        ThreadPool threadPool
+    ) throws IOException {
+        throw new UnsupportedOperationException("Not yet implemented");
+    }
+}

server/src/main/java/org/opensearch/storage/directory/TieredDirectory.java

@@ -0,0 +1,49 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.storage.directory;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.store.Directory;
+import org.opensearch.index.store.CompositeDirectory;
+import org.opensearch.index.store.remote.filecache.FileCache;
+import org.opensearch.storage.prefetch.TieredStoragePrefetchSettings;
+import org.opensearch.threadpool.ThreadPool;
+
+import java.util.function.Supplier;
+
+/**
+ * Extension of CompositeDirectory to support writable warm and other related features.
+ * Directory overrides (listAll, deleteFile, rename, openInput, close, sync, afterSyncToRemote),
+ * file caching, and full-file-to-block switching logic will be added in the implementation PR.
+ */
+public class TieredDirectory extends CompositeDirectory {
+
+    private static final Logger logger = LogManager.getLogger(TieredDirectory.class);
+    private final Supplier<TieredStoragePrefetchSettings> tieredStoragePrefetchSettingsSupplier;
+
+    /**
+     * Constructs a new TieredDirectory.
+     * @param localDirectory the local directory
+     * @param remoteDirectory the remote directory
+     * @param fileCache the file cache
+     * @param threadPool the thread pool
+     * @param tieredStoragePrefetchSettingsSupplier supplier for prefetch settings
+     */
+    public TieredDirectory(
+        Directory localDirectory,
+        Directory remoteDirectory,
+        FileCache fileCache,
+        ThreadPool threadPool,
+        Supplier<TieredStoragePrefetchSettings> tieredStoragePrefetchSettingsSupplier
+    ) {
+        super(localDirectory, remoteDirectory, fileCache, threadPool);
+        this.tieredStoragePrefetchSettingsSupplier = tieredStoragePrefetchSettingsSupplier;
+    }
+}

server/src/main/java/org/opensearch/storage/directory/TieredDirectoryFactory.java

@@ -0,0 +1,52 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.storage.directory;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.store.Directory;
+import org.opensearch.index.IndexSettings;
+import org.opensearch.index.shard.ShardPath;
+import org.opensearch.index.store.remote.filecache.FileCache;
+import org.opensearch.plugins.IndexStorePlugin;
+import org.opensearch.storage.prefetch.TieredStoragePrefetchSettings;
+import org.opensearch.threadpool.ThreadPool;
+
+import java.io.IOException;
+import java.util.function.Supplier;
+
+/**
+ * Factory to create TieredDirectory.
+ * The newDirectory implementation will be added in the implementation PR.
+ */
+public class TieredDirectoryFactory implements IndexStorePlugin.CompositeDirectoryFactory {
+
+    private static final Logger logger = LogManager.getLogger(TieredDirectoryFactory.class);
+    private final Supplier<TieredStoragePrefetchSettings> tieredStoragePrefetchSettingsSupplier;
+
+    /**
+     * Constructs a new TieredDirectoryFactory.
+     * @param tieredStoragePrefetchSettingsSupplier supplier for prefetch settings
+     */
+    public TieredDirectoryFactory(Supplier<TieredStoragePrefetchSettings> tieredStoragePrefetchSettingsSupplier) {
+        this.tieredStoragePrefetchSettingsSupplier = tieredStoragePrefetchSettingsSupplier;
+    }
+
+    @Override
+    public Directory newDirectory(
+        IndexSettings indexSettings,
+        ShardPath shardPath,
+        IndexStorePlugin.DirectoryFactory directoryFactory,
+        Directory directory,
+        FileCache fileCache,
+        ThreadPool threadPool
+    ) throws IOException {
+        throw new UnsupportedOperationException("Not yet implemented");
+    }
+}

server/src/main/java/org/opensearch/storage/directory/package-info.java

@@ -0,0 +1,10 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/** Directory implementations for tiered storage. */
+package org.opensearch.storage.directory;