feat(storage): object migration methods

package.json

@@ -12,11 +12,7 @@
   ],
   "scripts": {
     "build": "npm run build --workspaces --if-present",
-    "build:storage": "npm run build --workspace=@tigrisdata/storage",
-    "build:iam": "npm run build --workspace=@tigrisdata/iam",
-    "build:keyv-tigris": "npm run build --workspace=@tigrisdata/keyv-tigris",
-    "build:react": "npm run build --workspace=@tigrisdata/react",
-    "dev": "npm run dev --workspaces",
+    "dev": "npm run dev --workspaces --if-present",
     "publint": "npm run publint --workspaces --if-present",
     "lint": "npm run lint --workspaces --if-present",
     "lint:fix": "npm run lint:fix --workspaces --if-present",

packages/iam/package.json

@@ -27,8 +27,8 @@
     "access": "public"
   },
   "scripts": {
-    "build": "tsup",
-    "dev": "tsup --watch",
+    "build": "tsc --noEmit && tsup",
+    "dev": "tsc --noEmit && tsup --watch",
     "publint": "publint",
     "lint": "eslint src/**/*.ts",
     "lint:fix": "eslint src/**/*.ts --fix",

packages/keyv-tigris/package.json

@@ -27,8 +27,8 @@
     "access": "public"
   },
   "scripts": {
-    "build": "tsup",
-    "dev": "tsup --watch",
+    "build": "tsc --noEmit && tsup",
+    "dev": "tsc --noEmit && tsup --watch",
     "clean": "rm -rf dist",
     "test": "vitest run",
     "publint": "publint"

packages/react/package.json

@@ -47,8 +47,8 @@
   },
   "homepage": "https://github.com/tigrisdata/storage/tree/main/packages/react#readme",
   "scripts": {
-    "build": "tsup",
-    "dev": "tsup --watch",
+    "build": "tsc --noEmit && tsup",
+    "dev": "tsc --noEmit && tsup --watch",
     "test": "vitest run --config vitest.config.ts",
     "prepublishOnly": "npm run build"
   },

packages/storage/package.json

@@ -37,8 +37,8 @@
     "access": "public"
   },
   "scripts": {
-    "build": "tsup",
-    "dev": "tsup --watch",
+    "build": "tsc --noEmit && tsup",
+    "dev": "tsc --noEmit && tsup --watch",
     "publint": "publint",
     "lint": "eslint 'src/**/*.ts'",
     "lint:fix": "eslint 'src/**/*.ts' --fix",

packages/storage/src/lib/config.ts

@@ -5,7 +5,7 @@ import {
 } from '@shared/index';
 import type { TigrisStorageConfig } from './types';
 
-const configMap: Record<keyof TigrisStorageConfig, string> = {
+const configMap: Partial<Record<keyof TigrisStorageConfig, string>> = {
   endpoint: 'TIGRIS_STORAGE_ENDPOINT',
   bucket: 'TIGRIS_STORAGE_BUCKET',
   accessKeyId: 'TIGRIS_STORAGE_ACCESS_KEY_ID',

packages/storage/src/lib/object/get.ts

@@ -1,6 +1,6 @@
 import { GetObjectCommand } from '@aws-sdk/client-s3';
 import type { HttpRequest } from '@aws-sdk/types';
-import { TigrisHeaders } from '@shared/index';
+import { handleError, TigrisHeaders } from '@shared/index';
 import { config } from '../config';
 import { createTigrisClient } from '../tigris-client';
 import type { TigrisStorageConfig, TigrisStorageResponse } from '../types';
@@ -104,27 +104,3 @@ export async function get(
     return handleError(error as Error);
   }
 }
-
-const handleError = (error: Error) => {
-  let errorMessage: string | undefined;
-
-  if ((error as { Code?: string }).Code === 'AccessDenied') {
-    errorMessage =
-      'Access denied while downloading from Tigris Storage. Please check your credentials.';
-  }
-  if ((error as { Code?: string }).Code === 'NoSuchKey') {
-    errorMessage = 'File not found in Tigris Storage';
-  }
-
-  if (errorMessage) {
-    return {
-      error: new Error(errorMessage),
-    };
-  }
-
-  return {
-    error: new Error(
-      error?.message || 'Unexpected error while downloading from Tigris Storage'
-    ),
-  };
-};

packages/storage/src/lib/object/list.ts

@@ -11,6 +11,7 @@ export type ListOptions = {
   limit?: number;
   paginationToken?: string;
   snapshotVersion?: string;
+  source?: 'tigris' | 'shadow';
   config?: TigrisStorageConfig;
 };
 
@@ -23,6 +24,7 @@ export type ListItem = {
 
 export type ListResponse = {
   items: ListItem[];
+  commonPrefixes: string[];
   paginationToken: string | undefined;
   hasMore: boolean;
 };
@@ -65,6 +67,18 @@ export async function list(
     );
   }
 
+  if (options?.source) {
+    const source = options.source;
+    list.middlewareStack.add(
+      (next) => async (args) => {
+        const req = args.request as HttpRequest;
+        req.headers[TigrisHeaders.BUCKET_LIST_SOURCE] = source;
+        return next(args);
+      },
+      { name: 'X-Tigris-List-Source-Middleware', step: 'build', override: true }
+    );
+  }
+
   try {
     return tigrisClient
       .send(list)
@@ -78,6 +92,9 @@ export async function list(
                 size: item.Size ?? 0,
                 lastModified: item.LastModified ?? new Date(),
               })) ?? [],
+            commonPrefixes:
+              res.CommonPrefixes?.map((p) => p.Prefix ?? '').filter(Boolean) ??
+              [],
             paginationToken: res.NextContinuationToken,
             hasMore: res.IsTruncated ?? false,
           },

packages/storage/src/lib/object/migrate.ts

@@ -0,0 +1,94 @@
+import { HeadObjectCommand } from '@aws-sdk/client-s3';
+import type { HttpRequest, HttpResponse } from '@aws-sdk/types';
+import { handleError, TigrisHeaders } from '@shared/index';
+import { config } from '../config';
+import { createTigrisClient } from '../tigris-client';
+import type { TigrisStorageConfig, TigrisStorageResponse } from '../types';
+
+export type MigrateOptions = {
+  config?: TigrisStorageConfig;
+};
+
+export async function migrate(
+  path: string,
+  options?: MigrateOptions
+): Promise<TigrisStorageResponse<void, Error>> {
+  const { data: tigrisClient, error } = createTigrisClient(options?.config);
+
+  if (error) {
+    return { error };
+  }
+
+  const head = new HeadObjectCommand({
+    Bucket: options?.config?.bucket ?? config.bucket,
+    Key: path,
+  });
+
+  head.middlewareStack.add(
+    (next) => async (args) => {
+      const req = args.request as HttpRequest;
+      req.headers[TigrisHeaders.SCHEDULE_MIGRATION] = 'true';
+      const result = await next(args);
+      return result;
+    },
+    {
+      step: 'build',
+    }
+  );
+
+  try {
+    return tigrisClient
+      .send(head)
+      .then(async () => {
+        return {
+          data: undefined,
+        };
+      })
+      .catch(handleError);
+  } catch (error) {
+    return handleError(error as Error);
+  }
+}
+
+export async function isMigrated(
+  path: string,
+  options?: MigrateOptions
+): Promise<TigrisStorageResponse<boolean, Error>> {
+  const { data: tigrisClient, error } = createTigrisClient(options?.config);
+
+  if (error) {
+    return { error };
+  }
+
+  const head = new HeadObjectCommand({
+    Bucket: options?.config?.bucket ?? config.bucket,
+    Key: path,
+  });
+
+  let responseHeaders: Record<string, string> = {};
+
+  head.middlewareStack.add(
+    (next) => async (args) => {
+      const result = await next(args);
+      responseHeaders = (result.response as HttpResponse).headers;
+
+      return result;
+    },
+    {
+      step: 'deserialize',
+    }
+  );
+
+  try {
+    return tigrisClient
+      .send(head)
+      .then(async () => {
+        return {
+          data: TigrisHeaders.SERVED_FROM.toLowerCase() in responseHeaders,
+        };
+      })
+      .catch(handleError);
+  } catch (error) {
+    return handleError(error as Error);
+  }
+}

packages/storage/src/lib/object/remove.ts

@@ -2,6 +2,7 @@ import { DeleteObjectCommand } from '@aws-sdk/client-s3';
 import { config } from '../config';
 import { createTigrisClient } from '../tigris-client';
 import type { TigrisStorageConfig, TigrisStorageResponse } from '../types';
+import { handleError } from '@shared/utils';
 
 export type RemoveOptions = {
   config?: TigrisStorageConfig;
@@ -29,17 +30,6 @@ export async function remove(
       })
       .catch(handleError);
   } catch (error) {
-    return handleError(error);
+    return handleError(error as Error);
   }
 }
-
-const handleError = (error: unknown) => {
-  if ((error as { Code?: string }).Code === 'AccessDenied') {
-    return {
-      error: new Error('Access denied while deleting file'),
-    };
-  }
-  return {
-    error: new Error('Error deleting file'),
-  };
-};

packages/storage/src/lib/object/update.ts

@@ -3,8 +3,8 @@ import { createTigrisClient } from '../tigris-client';
 import { TigrisHeaders } from '@shared/headers';
 import type { TigrisStorageConfig, TigrisStorageResponse } from '../types';
 import { config, missingConfigError } from '../config';
-import { handleError } from '../utils';
 import { createStorageClient } from '../http-client';
+import { handleError } from '@shared/utils';
 
 export type UpdateObjectOptions = {
   config?: TigrisStorageConfig;

packages/storage/src/lib/utils.ts

@@ -4,26 +4,3 @@ export const addRandomSuffix = (path: string) => {
   const baseName = pathParts.join('.');
   return `${baseName}-${new Date().getTime().toString(36) + Math.random().toString(36).substring(2, 8)}${extension ? `.${extension}` : ''}`;
 };
-
-export const handleError = (error: Error) => {
-  let errorMessage: string | undefined;
-
-  if ((error as { Code?: string }).Code === 'AccessDenied') {
-    errorMessage = 'Access denied. Please check your credentials.';
-  }
-  if ((error as { Code?: string }).Code === 'NoSuchKey') {
-    errorMessage = 'File not found in Tigris Storage';
-  }
-
-  if (errorMessage) {
-    return {
-      error: new Error(errorMessage),
-    };
-  }
-
-  return {
-    error: new Error(
-      error?.message || 'Unexpected error while processing request'
-    ),
-  };
-};

packages/storage/src/server.ts

@@ -36,9 +36,9 @@ export { setBucketTtl, type SetBucketTtlOptions } from './lib/bucket/set/ttl';
 export {
   createBucketSnapshot,
   listBucketSnapshots,
+  type BucketSnapshot,
   type CreateBucketSnapshotOptions,
   type CreateBucketSnapshotResponse,
-  type BucketSnapshot,
   type ListBucketSnapshotsOptions,
   type ListBucketSnapshotsResponse,
 } from './lib/bucket/snapshot';
@@ -53,6 +53,11 @@ export {
   type UpdateBucketResponse,
 } from './lib/bucket/types';
 export { updateBucket, type UpdateBucketOptions } from './lib/bucket/update';
+export {
+  bundle,
+  type BundleOptions,
+  type BundleResponse,
+} from './lib/object/bundle';
 export { get, type GetOptions, type GetResponse } from './lib/object/get';
 export { head, type HeadOptions, type HeadResponse } from './lib/object/head';
 export {
@@ -61,6 +66,7 @@ export {
   type ListOptions,
   type ListResponse,
 } from './lib/object/list';
+export { isMigrated, migrate, type MigrateOptions } from './lib/object/migrate';
 export {
   completeMultipartUpload,
   getPartsPresignedUrls,
@@ -93,9 +99,4 @@ export {
   handleClientUpload,
   type ClientUploadRequest,
 } from './lib/upload/server';
-export {
-  bundle,
-  type BundleOptions,
-  type BundleResponse,
-} from './lib/object/bundle';
 export { UploadAction } from './lib/upload/shared';

shared/headers.ts

@@ -6,13 +6,18 @@ export enum TigrisHeaders {
   NAMESPACE = 'X-Tigris-Namespace',
   STORAGE_CLASS = 'X-Amz-Storage-Class',
 
+  BUCKET_LIST_SOURCE = 'X-Tigris-List-Source', // tigris or shadow
+  SCHEDULE_MIGRATION = 'X-Tigris-Schedule-Migration',
+  SERVED_FROM = 'X-Tigris-Served-From',
+
   REGIONS = 'X-Tigris-Regions',
   SNAPSHOT = 'X-Tigris-Snapshot',
   SNAPSHOT_VERSION = 'X-Tigris-Snapshot-Version',
   SNAPSHOT_ENABLED = 'X-Tigris-Enable-Snapshot',
   HAS_FORKS = 'X-Tigris-Is-Fork-Parent',
   FORK_SOURCE_BUCKET = 'X-Tigris-Fork-Source-Bucket',
   FORK_SOURCE_BUCKET_SNAPSHOT = 'X-Tigris-Fork-Source-Bucket-Snapshot',
+
   RENAME = 'X-Tigris-Rename',
   COPY_SOURCE = 'X-Amz-Copy-Source',
   BUNDLE_FORMAT = 'X-Tigris-Bundle-Format',

shared/http-client.ts

@@ -48,7 +48,7 @@ const cachedHttpClients = new Map<string, TigrisHttpClient>();
 /**
  * Generate AWS Signature V4 headers for a request
  */
-export async function generateSignatureHeaders(
+async function generateSignatureHeaders(
   method: string,
   url: URL,
   headers: Record<string, string>,

shared/index.ts

@@ -4,7 +4,6 @@ export { executeWithConcurrency, handleError, toError } from './utils';
 export { TigrisHeaders } from './headers';
 export {
   createTigrisHttpClient,
-  generateSignatureHeaders,
   type HttpClientRequest,
   type HttpClientResponse,
   type TigrisHttpClient,