diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ce79d9d2..c04230d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,10 +18,10 @@ jobs: lint: timeout-minutes: 10 name: lint - runs-on: ${{ github.repository == 'stainless-sdks/reductoai-python' && 'depot-ubuntu-24.04' || 'ubuntu-latest' }} + runs-on: ${{ startsWith(github.repository, 'stainless-sdks/') && 'depot-ubuntu-24.04' || 'ubuntu-latest' }} if: (github.event_name == 'push' || github.event.pull_request.head.repo.fork) && (github.event_name != 'push' || github.event.head_commit.message != 'codegen metadata') steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install Rye run: | @@ -44,9 +44,9 @@ jobs: permissions: contents: read id-token: write - runs-on: ${{ github.repository == 'stainless-sdks/reductoai-python' && 'depot-ubuntu-24.04' || 'ubuntu-latest' }} + runs-on: ${{ startsWith(github.repository, 'stainless-sdks/') && 'depot-ubuntu-24.04' || 'ubuntu-latest' }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install Rye run: | @@ -67,7 +67,7 @@ jobs: github.repository == 'stainless-sdks/reductoai-python' && !startsWith(github.ref, 'refs/heads/stl/') id: github-oidc - uses: actions/github-script@v8 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 with: script: core.setOutput('github_token', await core.getIDToken()); @@ -84,10 +84,10 @@ jobs: test: timeout-minutes: 10 name: test - runs-on: ${{ github.repository == 'stainless-sdks/reductoai-python' && 'depot-ubuntu-24.04' || 'ubuntu-latest' }} + runs-on: ${{ startsWith(github.repository, 'stainless-sdks/') && 'depot-ubuntu-24.04' || 'ubuntu-latest' }} if: github.event_name == 'push' || github.event.pull_request.head.repo.fork steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install Rye run: | diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 269cd72f..fe4fe30a 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install Rye run: | diff --git a/.github/workflows/release-doctor.yml b/.github/workflows/release-doctor.yml index 66a180cf..f3b41079 100644 --- a/.github/workflows/release-doctor.yml +++ b/.github/workflows/release-doctor.yml @@ -12,7 +12,7 @@ jobs: if: github.repository == 'reductoai/reducto-python-sdk' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch' || startsWith(github.head_ref, 'release-please') || github.head_ref == 'next') steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Check release environment run: | diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 0c2ecec6..86b0e83d 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "0.20.0" + ".": "0.21.0" } \ No newline at end of file diff --git a/.stats.yml b/.stats.yml index 40b629f2..5fad2791 100644 --- a/.stats.yml +++ b/.stats.yml @@ -1,4 +1,4 @@ configured_endpoints: 17 -openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/reducto%2Freductoai-1b16fbf5337f188d0b66a5992f0d241be80c46c45412ef9830cb19b11437d1c6.yml -openapi_spec_hash: 88f89b5803058bfa20d5da05c2bcf754 -config_hash: 9dd1f73da997aefc8516b226e0e7fed7 +openapi_spec_url: https://storage.googleapis.com/stainless-sdk-openapi-specs/reducto/reductoai-8c7080fc66341e6b6345375240e332bddea0a6f014e0e453aa08432a4b054a98.yml +openapi_spec_hash: b961adc626eb9d33d75d8cc8bcb49d05 +config_hash: 9fa10baf03f994be027bf73b29ac8572 diff --git a/README.md b/README.md index 16329cb2..055265b1 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,15 @@ and offers both synchronous and asynchronous clients powered by [httpx](https:// It is generated with [Stainless](https://www.stainless.com/). +## MCP Server + +Use the Reducto MCP Server to enable AI assistants to interact with this API, allowing them to explore endpoints, make test requests, and use documentation to help integrate this SDK into your application. + +[![Add to Cursor](https://cursor.com/deeplink/mcp-install-dark.svg)](https://cursor.com/en-US/install-mcp?name=reductoai-mcp&config=eyJjb21tYW5kIjoibnB4IiwiYXJncyI6WyIteSIsInJlZHVjdG9haS1tY3AiXSwiZW52Ijp7IlJFRFVDVE9fQVBJX0tFWSI6Ik15IEFQSSBLZXkifX0) +[![Install in VS Code](https://img.shields.io/badge/_-Add_to_VS_Code-blue?style=for-the-badge&logo=data:image/svg%2bxml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIGZpbGw9Im5vbmUiIHZpZXdCb3g9IjAgMCA0MCA0MCI+PHBhdGggZmlsbD0iI0VFRSIgZmlsbC1ydWxlPSJldmVub2RkIiBkPSJNMzAuMjM1IDM5Ljg4NGEyLjQ5MSAyLjQ5MSAwIDAgMS0xLjc4MS0uNzNMMTIuNyAyNC43OGwtMy40NiAyLjYyNC0zLjQwNiAyLjU4MmExLjY2NSAxLjY2NSAwIDAgMS0xLjA4Mi4zMzggMS42NjQgMS42NjQgMCAwIDEtMS4wNDYtLjQzMWwtMi4yLTJhMS42NjYgMS42NjYgMCAwIDEgMC0yLjQ2M0w3LjQ1OCAyMCA0LjY3IDE3LjQ1MyAxLjUwNyAxNC41N2ExLjY2NSAxLjY2NSAwIDAgMSAwLTIuNDYzbDIuMi0yYTEuNjY1IDEuNjY1IDAgMCAxIDIuMTMtLjA5N2w2Ljg2MyA1LjIwOUwyOC40NTIuODQ0YTIuNDg4IDIuNDg4IDAgMCAxIDEuODQxLS43MjljLjM1MS4wMDkuNjk5LjA5MSAxLjAxOS4yNDVsOC4yMzYgMy45NjFhMi41IDIuNSAwIDAgMSAxLjQxNSAyLjI1M3YuMDk5LS4wNDVWMzMuMzd2LS4wNDUuMDk1YTIuNTAxIDIuNTAxIDAgMCAxLTEuNDE2IDIuMjU3bC04LjIzNSAzLjk2MWEyLjQ5MiAyLjQ5MiAwIDAgMS0xLjA3Ny4yNDZabS43MTYtMjguOTQ3LTExLjk0OCA5LjA2MiAxMS45NTIgOS4wNjUtLjAwNC0xOC4xMjdaIi8+PC9zdmc+)](https://vscode.stainless.com/mcp/%7B%22name%22%3A%22reductoai-mcp%22%2C%22command%22%3A%22npx%22%2C%22args%22%3A%5B%22-y%22%2C%22reductoai-mcp%22%5D%2C%22env%22%3A%7B%22REDUCTO_API_KEY%22%3A%22My%20API%20Key%22%7D%7D) + +> Note: You may need to set environment variables in your MCP client. + ## Documentation The REST API documentation can be found on [docs.reductoai.com](https://docs.reductoai.com). The full API of this library can be found in [api.md](api.md). @@ -129,6 +138,23 @@ response = client.parse.run( print(response.enhance) ``` +## File uploads + +Request parameters that correspond to file uploads can be passed as `bytes`, or a [`PathLike`](https://docs.python.org/3/library/os.html#os.PathLike) instance or a tuple of `(filename, contents, media type)`. + +```python +from pathlib import Path +from reducto import Reducto + +client = Reducto() + +client.upload( + file=Path("/path/to/file"), +) +``` + +The async client uses the exact same interface. If you pass a [`PathLike`](https://docs.python.org/3/library/os.html#os.PathLike) instance, the file contents will be read asynchronously automatically. + ## Handling errors When the library is unable to connect to the API (for example, due to network connection problems or a timeout), a subclass of `reducto.APIConnectionError` is raised. diff --git a/api.md b/api.md index f0f17287..77f2b044 100644 --- a/api.md +++ b/api.md @@ -1,7 +1,29 @@ # Shared Types ```python -from reducto.types import Upload +from reducto.types import ( + AsyncEditResponse, + AsyncExtractResponse, + AsyncParseResponse, + AsyncPipelineResponse, + AsyncSplitResponse, + Chunking, + ClassifyResponse, + DirectWebhookConfig, + EditResponse, + ExtractResponse, + FigureAgentic, + PageRange, + ParseResponse, + PipelineResponse, + SplitLargeTables, + SplitResponse, + SvixWebhookConfig, + TableAgentic, + TextAgentic, + Upload, + WebhookConfigNew, +) ``` # Reducto @@ -25,10 +47,8 @@ Types: from reducto.types import ( AsyncConfigV3, AsyncParseConfig, - AsyncParseResponse, Enhance, Formatting, - ParseResponse, Retrieval, Settings, Spreadsheet, @@ -39,7 +59,7 @@ from reducto.types import ( Methods: - client.parse.run(\*\*params) -> ParseRunResponse -- client.parse.run_job(\*\*params) -> AsyncParseResponse +- client.parse.run_job(\*\*params) -> AsyncParseResponse # Extract @@ -48,7 +68,6 @@ Types: ```python from reducto.types import ( AsyncExtractConfig, - AsyncExtractResponse, ExtractSettings, ExtractUsage, Instructions, @@ -61,65 +80,52 @@ from reducto.types import ( Methods: - client.extract.run(\*\*params) -> ExtractRunResponse -- client.extract.run_job(\*\*params) -> AsyncExtractResponse +- client.extract.run_job(\*\*params) -> AsyncExtractResponse # Split Types: ```python -from reducto.types import ( - DeepSplitPageEvidence, - ParseUsage, - SplitCategory, - SplitResponse, - SplitTableOptions, - SplitRunJobResponse, -) +from reducto.types import DeepSplitPageEvidence, ParseUsage, SplitCategory ``` Methods: -- client.split.run(\*\*params) -> SplitResponse -- client.split.run_job(\*\*params) -> SplitRunJobResponse +- client.split.run(\*\*params) -> SplitResponse +- client.split.run_job(\*\*params) -> AsyncSplitResponse # Edit Types: ```python -from reducto.types import BoundingBox, EditOptions, EditResponse, EditWidget, EditRunJobResponse +from reducto.types import BoundingBox, EditOptions, EditWidget ``` Methods: -- client.edit.run(\*\*params) -> EditResponse -- client.edit.run_job(\*\*params) -> EditRunJobResponse +- client.edit.run(\*\*params) -> EditResponse +- client.edit.run_job(\*\*params) -> AsyncEditResponse # Pipeline Types: ```python -from reducto.types import PipelineResponse, PipelineSettings, PipelineRunJobResponse +from reducto.types import PipelineSettings ``` Methods: -- client.pipeline.run(\*\*params) -> PipelineResponse -- client.pipeline.run_job(\*\*params) -> PipelineRunJobResponse +- client.pipeline.run(\*\*params) -> PipelineResponse +- client.pipeline.run_job(\*\*params) -> AsyncPipelineResponse # Classify -Types: - -```python -from reducto.types import ClassifyResponse, PageRange -``` - Methods: -- client.classify.run(\*\*params) -> ClassifyResponse +- client.classify.run(\*\*params) -> ClassifyResponse # Webhook @@ -138,7 +144,7 @@ Methods: Types: ```python -from reducto.types import ExtractResponse, JobGetResponse, JobGetAllResponse +from reducto.types import JobGetResponse, JobGetAllResponse ``` Methods: diff --git a/pyproject.toml b/pyproject.toml index 22d7bf0e..354ab96e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "reductoai" -version = "0.20.0" +version = "0.21.0" description = "The official Python library for the reducto API" dynamic = ["readme"] license = "Apache-2.0" @@ -168,7 +168,7 @@ show_error_codes = true # # We also exclude our `tests` as mypy doesn't always infer # types correctly and Pyright will still catch any type errors. -exclude = ['src/reducto/_files.py', '_dev/.*.py', 'tests/.*'] +exclude = ["src/reducto/_files.py", "_dev/.*.py", "tests/.*"] strict_equality = true implicit_reexport = true diff --git a/scripts/bootstrap b/scripts/bootstrap index b430fee3..fe8451e4 100755 --- a/scripts/bootstrap +++ b/scripts/bootstrap @@ -4,7 +4,7 @@ set -e cd "$(dirname "$0")/.." -if [ -f "Brewfile" ] && [ "$(uname -s)" = "Darwin" ] && [ "$SKIP_BREW" != "1" ] && [ -t 0 ]; then +if [ -f "Brewfile" ] && [ "$(uname -s)" = "Darwin" ] && [ "${SKIP_BREW:-}" != "1" ] && [ -t 0 ]; then brew bundle check >/dev/null 2>&1 || { echo -n "==> Install Homebrew dependencies? (y/N): " read -r response diff --git a/src/reducto/_base_client.py b/src/reducto/_base_client.py index 1e31cb4d..ee1f4314 100644 --- a/src/reducto/_base_client.py +++ b/src/reducto/_base_client.py @@ -558,6 +558,10 @@ def _build_request( files = cast(HttpxRequestFiles, ForceMultipartDict()) prepared_url = self._prepare_url(options.url) + # preserve hard-coded query params from the url + if params and prepared_url.query: + params = {**dict(prepared_url.params.items()), **params} + prepared_url = prepared_url.copy_with(raw_path=prepared_url.raw_path.split(b"?", 1)[0]) if "_" in prepared_url.host: # work around https://github.com/encode/httpx/discussions/2880 kwargs["extensions"] = {"sni_hostname": prepared_url.host.replace("_", "-")} diff --git a/src/reducto/_client.py b/src/reducto/_client.py index 901feebe..bc72b815 100644 --- a/src/reducto/_client.py +++ b/src/reducto/_client.py @@ -11,6 +11,7 @@ from . import _exceptions from ._qs import Querystring from .types import client_upload_params +from ._files import deepcopy_with_paths from ._types import ( Body, Omit, @@ -18,6 +19,7 @@ Headers, Timeout, NotGiven, + FileTypes, Transport, ProxiesTypes, RequestOptions, @@ -26,6 +28,8 @@ ) from ._utils import ( is_given, + is_mapping_t, + extract_files, maybe_transform, get_async_library, async_maybe_transform, @@ -147,6 +151,15 @@ def __init__( except KeyError as exc: raise ValueError(f"Unknown environment: {environment}") from exc + custom_headers_env = os.environ.get("REDUCTO_CUSTOM_HEADERS") + if custom_headers_env is not None: + parsed: dict[str, str] = {} + for line in custom_headers_env.split("\n"): + colon = line.find(":") + if colon >= 0: + parsed[line[:colon].strip()] = line[colon + 1 :].strip() + default_headers = {**parsed, **(default_headers if is_mapping_t(default_headers) else {})} + super().__init__( version=__version__, base_url=base_url, @@ -221,9 +234,11 @@ def qs(self) -> Querystring: @override def _auth_headers(self, security: SecurityOptions) -> dict[str, str]: - return { - **(self._skippable_http_bearer if security.get("skippable_http_bearer", False) else {}), - } + headers: dict[str, str] = {} + if security.get("skippable_http_bearer", False): + for key, value in self._skippable_http_bearer.items(): + headers.setdefault(key, value) + return headers @property def _skippable_http_bearer(self) -> dict[str, str]: @@ -315,7 +330,7 @@ def upload( self, *, extension: Optional[str] | Omit = omit, - file: Optional[str] | Omit = omit, + file: Optional[FileTypes] | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -335,9 +350,17 @@ def upload( timeout: Override the client-level default timeout for this request, in seconds """ + body = deepcopy_with_paths({"file": file}, [["file"]]) + files = extract_files(cast(Mapping[str, object], body), paths=[["file"]]) + if files: + # It should be noted that the actual Content-Type header that will be + # sent to the server will contain a `boundary` parameter, e.g. + # multipart/form-data; boundary=---abc-- + extra_headers = {"Content-Type": "multipart/form-data", **(extra_headers or {})} return self.post( "/upload", - body=maybe_transform({"file": file}, client_upload_params.ClientUploadParams), + body=maybe_transform(body, client_upload_params.ClientUploadParams), + files=files, options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, @@ -450,6 +473,15 @@ def __init__( except KeyError as exc: raise ValueError(f"Unknown environment: {environment}") from exc + custom_headers_env = os.environ.get("REDUCTO_CUSTOM_HEADERS") + if custom_headers_env is not None: + parsed: dict[str, str] = {} + for line in custom_headers_env.split("\n"): + colon = line.find(":") + if colon >= 0: + parsed[line[:colon].strip()] = line[colon + 1 :].strip() + default_headers = {**parsed, **(default_headers if is_mapping_t(default_headers) else {})} + super().__init__( version=__version__, base_url=base_url, @@ -524,9 +556,11 @@ def qs(self) -> Querystring: @override def _auth_headers(self, security: SecurityOptions) -> dict[str, str]: - return { - **(self._skippable_http_bearer if security.get("skippable_http_bearer", False) else {}), - } + headers: dict[str, str] = {} + if security.get("skippable_http_bearer", False): + for key, value in self._skippable_http_bearer.items(): + headers.setdefault(key, value) + return headers @property def _skippable_http_bearer(self) -> dict[str, str]: @@ -618,7 +652,7 @@ async def upload( self, *, extension: Optional[str] | Omit = omit, - file: Optional[str] | Omit = omit, + file: Optional[FileTypes] | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -638,9 +672,17 @@ async def upload( timeout: Override the client-level default timeout for this request, in seconds """ + body = deepcopy_with_paths({"file": file}, [["file"]]) + files = extract_files(cast(Mapping[str, object], body), paths=[["file"]]) + if files: + # It should be noted that the actual Content-Type header that will be + # sent to the server will contain a `boundary` parameter, e.g. + # multipart/form-data; boundary=---abc-- + extra_headers = {"Content-Type": "multipart/form-data", **(extra_headers or {})} return await self.post( "/upload", - body=await async_maybe_transform({"file": file}, client_upload_params.ClientUploadParams), + body=await async_maybe_transform(body, client_upload_params.ClientUploadParams), + files=files, options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, diff --git a/src/reducto/_files.py b/src/reducto/_files.py index cc14c14f..76c6988d 100644 --- a/src/reducto/_files.py +++ b/src/reducto/_files.py @@ -3,8 +3,8 @@ import io import os import pathlib -from typing import overload -from typing_extensions import TypeGuard +from typing import Sequence, cast, overload +from typing_extensions import TypeVar, TypeGuard import anyio @@ -17,7 +17,9 @@ HttpxFileContent, HttpxRequestFiles, ) -from ._utils import is_tuple_t, is_mapping_t, is_sequence_t +from ._utils import is_list, is_mapping, is_tuple_t, is_mapping_t, is_sequence_t + +_T = TypeVar("_T") def is_base64_file_input(obj: object) -> TypeGuard[Base64FileInput]: @@ -34,7 +36,7 @@ def assert_is_file_content(obj: object, *, key: str | None = None) -> None: if not is_file_content(obj): prefix = f"Expected entry at `{key}`" if key is not None else f"Expected file input `{obj!r}`" raise RuntimeError( - f"{prefix} to be bytes, an io.IOBase instance, PathLike or a tuple but received {type(obj)} instead." + f"{prefix} to be bytes, an io.IOBase instance, PathLike or a tuple but received {type(obj)} instead. See https://github.com/reductoai/reducto-python-sdk/tree/main#file-uploads" ) from None @@ -97,7 +99,7 @@ async def async_to_httpx_files(files: RequestFiles | None) -> HttpxRequestFiles elif is_sequence_t(files): files = [(key, await _async_transform_file(file)) for key, file in files] else: - raise TypeError("Unexpected file type input {type(files)}, expected mapping or sequence") + raise TypeError(f"Unexpected file type input {type(files)}, expected mapping or sequence") return files @@ -121,3 +123,51 @@ async def async_read_file_content(file: FileContent) -> HttpxFileContent: return await anyio.Path(file).read_bytes() return file + + +def deepcopy_with_paths(item: _T, paths: Sequence[Sequence[str]]) -> _T: + """Copy only the containers along the given paths. + + Used to guard against mutation by extract_files without copying the entire structure. + Only dicts and lists that lie on a path are copied; everything else + is returned by reference. + + For example, given paths=[["foo", "files", "file"]] and the structure: + { + "foo": { + "bar": {"baz": {}}, + "files": {"file": } + } + } + The root dict, "foo", and "files" are copied (they lie on the path). + "bar" and "baz" are returned by reference (off the path). + """ + return _deepcopy_with_paths(item, paths, 0) + + +def _deepcopy_with_paths(item: _T, paths: Sequence[Sequence[str]], index: int) -> _T: + if not paths: + return item + if is_mapping(item): + key_to_paths: dict[str, list[Sequence[str]]] = {} + for path in paths: + if index < len(path): + key_to_paths.setdefault(path[index], []).append(path) + + # if no path continues through this mapping, it won't be mutated and copying it is redundant + if not key_to_paths: + return item + + result = dict(item) + for key, subpaths in key_to_paths.items(): + if key in result: + result[key] = _deepcopy_with_paths(result[key], subpaths, index + 1) + return cast(_T, result) + if is_list(item): + array_paths = [path for path in paths if index < len(path) and path[index] == ""] + + # if no path expects a list here, nothing will be mutated inside it - return by reference + if not array_paths: + return cast(_T, item) + return cast(_T, [_deepcopy_with_paths(entry, array_paths, index + 1) for entry in item]) + return item diff --git a/src/reducto/_models.py b/src/reducto/_models.py index 1819e148..0ec36119 100644 --- a/src/reducto/_models.py +++ b/src/reducto/_models.py @@ -25,7 +25,9 @@ ClassVar, Protocol, Required, + Annotated, ParamSpec, + TypeAlias, TypedDict, TypeGuard, final, @@ -79,7 +81,15 @@ from ._constants import RAW_RESPONSE_HEADER if TYPE_CHECKING: + from pydantic import GetCoreSchemaHandler, ValidatorFunctionWrapHandler + from pydantic_core import CoreSchema, core_schema from pydantic_core.core_schema import ModelField, ModelSchema, LiteralSchema, ModelFieldsSchema +else: + try: + from pydantic_core import CoreSchema, core_schema + except ImportError: + CoreSchema = None + core_schema = None __all__ = ["BaseModel", "GenericModel"] @@ -396,6 +406,76 @@ def model_dump_json( ) +class _EagerIterable(list[_T], Generic[_T]): + """ + Accepts any Iterable[T] input (including generators), consumes it + eagerly, and validates all items upfront. + + Validation preserves the original container type where possible + (e.g. a set[T] stays a set[T]). Serialization (model_dump / JSON) + always emits a list — round-tripping through model_dump() will not + restore the original container type. + """ + + @classmethod + def __get_pydantic_core_schema__( + cls, + source_type: Any, + handler: GetCoreSchemaHandler, + ) -> CoreSchema: + (item_type,) = get_args(source_type) or (Any,) + item_schema: CoreSchema = handler.generate_schema(item_type) + list_of_items_schema: CoreSchema = core_schema.list_schema(item_schema) + + return core_schema.no_info_wrap_validator_function( + cls._validate, + list_of_items_schema, + serialization=core_schema.plain_serializer_function_ser_schema( + cls._serialize, + info_arg=False, + ), + ) + + @staticmethod + def _validate(v: Iterable[_T], handler: "ValidatorFunctionWrapHandler") -> Any: + original_type: type[Any] = type(v) + + # Normalize to list so list_schema can validate each item + if isinstance(v, list): + items: list[_T] = v + else: + try: + items = list(v) + except TypeError as e: + raise TypeError("Value is not iterable") from e + + # Validate items against the inner schema + validated: list[_T] = handler(items) + + # Reconstruct original container type + if original_type is list: + return validated + # str(list) produces the list's repr, not a string built from items, + # so skip reconstruction for str and its subclasses. + if issubclass(original_type, str): + return validated + try: + return original_type(validated) + except (TypeError, ValueError): + # If the type cannot be reconstructed, just return the validated list + return validated + + @staticmethod + def _serialize(v: Iterable[_T]) -> list[_T]: + """Always serialize as a list so Pydantic's JSON encoder is happy.""" + if isinstance(v, list): + return v + return list(v) + + +EagerIterable: TypeAlias = Annotated[Iterable[_T], _EagerIterable] + + def _construct_field(value: object, field: FieldInfo, key: str) -> object: if value is None: return field_get_default(field) diff --git a/src/reducto/_qs.py b/src/reducto/_qs.py index ada6fd3f..4127c19c 100644 --- a/src/reducto/_qs.py +++ b/src/reducto/_qs.py @@ -2,17 +2,13 @@ from typing import Any, List, Tuple, Union, Mapping, TypeVar from urllib.parse import parse_qs, urlencode -from typing_extensions import Literal, get_args +from typing_extensions import get_args -from ._types import NotGiven, not_given +from ._types import NotGiven, ArrayFormat, NestedFormat, not_given from ._utils import flatten _T = TypeVar("_T") - -ArrayFormat = Literal["comma", "repeat", "indices", "brackets"] -NestedFormat = Literal["dots", "brackets"] - PrimitiveData = Union[str, int, float, bool, None] # this should be Data = Union[PrimitiveData, "List[Data]", "Tuple[Data]", "Mapping[str, Data]"] # https://github.com/microsoft/pyright/issues/3555 @@ -101,7 +97,10 @@ def _stringify_item( items.extend(self._stringify_item(key, item, opts)) return items elif array_format == "indices": - raise NotImplementedError("The array indices format is not supported yet") + items = [] + for i, item in enumerate(value): + items.extend(self._stringify_item(f"{key}[{i}]", item, opts)) + return items elif array_format == "brackets": items = [] key = key + "[]" diff --git a/src/reducto/_types.py b/src/reducto/_types.py index 6c70e52c..1cd25278 100644 --- a/src/reducto/_types.py +++ b/src/reducto/_types.py @@ -47,6 +47,9 @@ ModelT = TypeVar("ModelT", bound=pydantic.BaseModel) _T = TypeVar("_T") +ArrayFormat = Literal["comma", "repeat", "indices", "brackets"] +NestedFormat = Literal["dots", "brackets"] + # Approximates httpx internal ProxiesTypes and RequestFiles types # while adding support for `PathLike` instances diff --git a/src/reducto/_utils/__init__.py b/src/reducto/_utils/__init__.py index 10cb66d2..1c090e51 100644 --- a/src/reducto/_utils/__init__.py +++ b/src/reducto/_utils/__init__.py @@ -24,7 +24,6 @@ coerce_integer as coerce_integer, file_from_path as file_from_path, strip_not_given as strip_not_given, - deepcopy_minimal as deepcopy_minimal, get_async_library as get_async_library, maybe_coerce_float as maybe_coerce_float, get_required_header as get_required_header, diff --git a/src/reducto/_utils/_utils.py b/src/reducto/_utils/_utils.py index eec7f4a1..199cd231 100644 --- a/src/reducto/_utils/_utils.py +++ b/src/reducto/_utils/_utils.py @@ -17,11 +17,11 @@ ) from pathlib import Path from datetime import date, datetime -from typing_extensions import TypeGuard +from typing_extensions import TypeGuard, get_args import sniffio -from .._types import Omit, NotGiven, FileTypes, HeadersLike +from .._types import Omit, NotGiven, FileTypes, ArrayFormat, HeadersLike _T = TypeVar("_T") _TupleT = TypeVar("_TupleT", bound=Tuple[object, ...]) @@ -40,25 +40,45 @@ def extract_files( query: Mapping[str, object], *, paths: Sequence[Sequence[str]], + array_format: ArrayFormat = "brackets", ) -> list[tuple[str, FileTypes]]: """Recursively extract files from the given dictionary based on specified paths. A path may look like this ['foo', 'files', '', 'data']. + ``array_format`` controls how ```` segments contribute to the emitted + field name. Supported values: ``"brackets"`` (``foo[]``), ``"repeat"`` and + ``"comma"`` (``foo``), ``"indices"`` (``foo[0]``, ``foo[1]``). + Note: this mutates the given dictionary. """ files: list[tuple[str, FileTypes]] = [] for path in paths: - files.extend(_extract_items(query, path, index=0, flattened_key=None)) + files.extend(_extract_items(query, path, index=0, flattened_key=None, array_format=array_format)) return files +def _array_suffix(array_format: ArrayFormat, array_index: int) -> str: + if array_format == "brackets": + return "[]" + if array_format == "indices": + return f"[{array_index}]" + if array_format == "repeat" or array_format == "comma": + # Both repeat the bare field name for each file part; there is no + # meaningful way to comma-join binary parts. + return "" + raise NotImplementedError( + f"Unknown array_format value: {array_format}, choose from {', '.join(get_args(ArrayFormat))}" + ) + + def _extract_items( obj: object, path: Sequence[str], *, index: int, flattened_key: str | None, + array_format: ArrayFormat, ) -> list[tuple[str, FileTypes]]: try: key = path[index] @@ -75,9 +95,11 @@ def _extract_items( if is_list(obj): files: list[tuple[str, FileTypes]] = [] - for entry in obj: - assert_is_file_content(entry, key=flattened_key + "[]" if flattened_key else "") - files.append((flattened_key + "[]", cast(FileTypes, entry))) + for array_index, entry in enumerate(obj): + suffix = _array_suffix(array_format, array_index) + emitted_key = (flattened_key + suffix) if flattened_key else suffix + assert_is_file_content(entry, key=emitted_key) + files.append((emitted_key, cast(FileTypes, entry))) return files assert_is_file_content(obj, key=flattened_key) @@ -86,8 +108,9 @@ def _extract_items( index += 1 if is_dict(obj): try: - # We are at the last entry in the path so we must remove the field - if (len(path)) == index: + # Remove the field if there are no more dict keys in the path, + # only "" traversal markers or end. + if all(p == "" for p in path[index:]): item = obj.pop(key) else: item = obj[key] @@ -105,6 +128,7 @@ def _extract_items( path, index=index, flattened_key=flattened_key, + array_format=array_format, ) elif is_list(obj): if key != "": @@ -116,9 +140,12 @@ def _extract_items( item, path, index=index, - flattened_key=flattened_key + "[]" if flattened_key is not None else "[]", + flattened_key=( + (flattened_key if flattened_key is not None else "") + _array_suffix(array_format, array_index) + ), + array_format=array_format, ) - for item in obj + for array_index, item in enumerate(obj) ] ) @@ -176,21 +203,6 @@ def is_iterable(obj: object) -> TypeGuard[Iterable[object]]: return isinstance(obj, Iterable) -def deepcopy_minimal(item: _T) -> _T: - """Minimal reimplementation of copy.deepcopy() that will only copy certain object types: - - - mappings, e.g. `dict` - - list - - This is done for performance reasons. - """ - if is_mapping(item): - return cast(_T, {k: deepcopy_minimal(v) for k, v in item.items()}) - if is_list(item): - return cast(_T, [deepcopy_minimal(entry) for entry in item]) - return item - - # copied from https://github.com/Rapptz/RoboDanny def human_join(seq: Sequence[str], *, delim: str = ", ", final: str = "or") -> str: size = len(seq) diff --git a/src/reducto/_version.py b/src/reducto/_version.py index 497850e3..30639123 100644 --- a/src/reducto/_version.py +++ b/src/reducto/_version.py @@ -1,4 +1,4 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. __title__ = "reducto" -__version__ = "0.20.0" # x-release-please-version +__version__ = "0.21.0" # x-release-please-version diff --git a/src/reducto/resources/classify.py b/src/reducto/resources/classify.py index 8702e75f..1b30af6e 100644 --- a/src/reducto/resources/classify.py +++ b/src/reducto/resources/classify.py @@ -18,7 +18,7 @@ async_to_streamed_response_wrapper, ) from .._base_client import make_request_options -from ..types.classify_response import ClassifyResponse +from ..types.shared.classify_response import ClassifyResponse __all__ = ["ClassifyResource", "AsyncClassifyResource"] @@ -49,8 +49,9 @@ def run( input: classify_run_params.Input, classification_schema: Iterable[classify_run_params.ClassificationSchema] | Omit = omit, document_metadata: Optional[str] | Omit = omit, + force_url_result: bool | Omit = omit, page_range: Optional[classify_run_params.PageRange] | Omit = omit, - persist_results: bool | Omit = omit, + priority: bool | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -58,28 +59,32 @@ def run( extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, ) -> ClassifyResponse: - """ - Classify + """Classify Args: - input: For parse/split/extract pipelines, the URL of the document to be processed. You - can provide one of the following: 1. A publicly available URL 2. A presigned S3 - URL 3. A reducto:// prefixed URL obtained from the /upload endpoint after - directly uploading a document 4. A jobid:// prefixed URL obtained from a - previous /parse invocation 5. A list of URLs (for multi-document pipelines, V3 - API only) + input: + The URL of the document to be classified. + + You can provide one of the following: - For edit pipelines, this should be a string containing the edit instructions + 1. A publicly available URL + 2. A presigned S3 URL + 3. A reducto:// prefixed URL obtained from the /upload endpoint after directly + uploading a document classification_schema: A list of classification categories and their matching criteria. document_metadata: Optional document-level metadata to include in classification prompts. + force_url_result: Force the endpoint result to be returned in URL form. + page_range: The page range to process (1-indexed). By default, the first 5 pages are used. - If more than 25 pages are selected, only the first 25 (after sorting) are used. - Only applies to PDFs; ignored for other document types. + At most 10 pages can be selected. Only applies to PDFs; ignored for other + document types. - persist_results: If True, persist the results indefinitely. Defaults to False. + priority: Workers poll the priority queue ahead of the standard queue, so priority jobs + start sooner when there is queued work; sync jobs are prioritized above async + jobs by default. extra_headers: Send extra headers @@ -96,8 +101,9 @@ def run( "input": input, "classification_schema": classification_schema, "document_metadata": document_metadata, + "force_url_result": force_url_result, "page_range": page_range, - "persist_results": persist_results, + "priority": priority, }, classify_run_params.ClassifyRunParams, ), @@ -134,8 +140,9 @@ async def run( input: classify_run_params.Input, classification_schema: Iterable[classify_run_params.ClassificationSchema] | Omit = omit, document_metadata: Optional[str] | Omit = omit, + force_url_result: bool | Omit = omit, page_range: Optional[classify_run_params.PageRange] | Omit = omit, - persist_results: bool | Omit = omit, + priority: bool | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -143,28 +150,32 @@ async def run( extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, ) -> ClassifyResponse: - """ - Classify + """Classify Args: - input: For parse/split/extract pipelines, the URL of the document to be processed. You - can provide one of the following: 1. A publicly available URL 2. A presigned S3 - URL 3. A reducto:// prefixed URL obtained from the /upload endpoint after - directly uploading a document 4. A jobid:// prefixed URL obtained from a - previous /parse invocation 5. A list of URLs (for multi-document pipelines, V3 - API only) + input: + The URL of the document to be classified. + + You can provide one of the following: - For edit pipelines, this should be a string containing the edit instructions + 1. A publicly available URL + 2. A presigned S3 URL + 3. A reducto:// prefixed URL obtained from the /upload endpoint after directly + uploading a document classification_schema: A list of classification categories and their matching criteria. document_metadata: Optional document-level metadata to include in classification prompts. + force_url_result: Force the endpoint result to be returned in URL form. + page_range: The page range to process (1-indexed). By default, the first 5 pages are used. - If more than 25 pages are selected, only the first 25 (after sorting) are used. - Only applies to PDFs; ignored for other document types. + At most 10 pages can be selected. Only applies to PDFs; ignored for other + document types. - persist_results: If True, persist the results indefinitely. Defaults to False. + priority: Workers poll the priority queue ahead of the standard queue, so priority jobs + start sooner when there is queued work; sync jobs are prioritized above async + jobs by default. extra_headers: Send extra headers @@ -181,8 +192,9 @@ async def run( "input": input, "classification_schema": classification_schema, "document_metadata": document_metadata, + "force_url_result": force_url_result, "page_range": page_range, - "persist_results": persist_results, + "priority": priority, }, classify_run_params.ClassifyRunParams, ), diff --git a/src/reducto/resources/edit.py b/src/reducto/resources/edit.py index 48cae916..c18274a4 100644 --- a/src/reducto/resources/edit.py +++ b/src/reducto/resources/edit.py @@ -18,10 +18,11 @@ async_to_streamed_response_wrapper, ) from .._base_client import make_request_options -from ..types.edit_response import EditResponse from ..types.edit_widget_param import EditWidgetParam from ..types.edit_options_param import EditOptionsParam -from ..types.edit_run_job_response import EditRunJobResponse +from ..types.shared.edit_response import EditResponse +from ..types.shared.async_edit_response import AsyncEditResponse +from ..types.shared_params.webhook_config_new import WebhookConfigNew __all__ = ["EditResource", "AsyncEditResource"] @@ -79,9 +80,9 @@ def run( form_schema: Form schema for PDF forms. List of widgets with their types, descriptions, and bounding boxes. Only works for PDFs. - priority: If True, attempts to process the job with priority if the user has priority - processing budget available; by default, sync jobs are prioritized above async - jobs. + priority: Workers poll the priority queue ahead of the standard queue, so priority jobs + start sooner when there is queued work; sync jobs are prioritized above async + jobs by default. extra_headers: Send extra headers @@ -117,14 +118,14 @@ def run_job( edit_options: EditOptionsParam | Omit = omit, form_schema: Optional[Iterable[EditWidgetParam]] | Omit = omit, priority: bool | Omit = omit, - webhook: edit_run_job_params.Webhook | Omit = omit, + webhook: WebhookConfigNew | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> EditRunJobResponse: + ) -> AsyncEditResponse: """Edit Async Args: @@ -143,9 +144,9 @@ def run_job( form_schema: Form schema for PDF forms. List of widgets with their types, descriptions, and bounding boxes. Only works for PDFs. - priority: If True, attempts to process the job with priority if the user has priority - processing budget available; by default, sync jobs are prioritized above async - jobs. + priority: Workers poll the priority queue ahead of the standard queue, so priority jobs + start sooner when there is queued work; sync jobs are prioritized above async + jobs by default. extra_headers: Send extra headers @@ -171,7 +172,7 @@ def run_job( options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=EditRunJobResponse, + cast_to=AsyncEditResponse, ) @@ -228,9 +229,9 @@ async def run( form_schema: Form schema for PDF forms. List of widgets with their types, descriptions, and bounding boxes. Only works for PDFs. - priority: If True, attempts to process the job with priority if the user has priority - processing budget available; by default, sync jobs are prioritized above async - jobs. + priority: Workers poll the priority queue ahead of the standard queue, so priority jobs + start sooner when there is queued work; sync jobs are prioritized above async + jobs by default. extra_headers: Send extra headers @@ -266,14 +267,14 @@ async def run_job( edit_options: EditOptionsParam | Omit = omit, form_schema: Optional[Iterable[EditWidgetParam]] | Omit = omit, priority: bool | Omit = omit, - webhook: edit_run_job_params.Webhook | Omit = omit, + webhook: WebhookConfigNew | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> EditRunJobResponse: + ) -> AsyncEditResponse: """Edit Async Args: @@ -292,9 +293,9 @@ async def run_job( form_schema: Form schema for PDF forms. List of widgets with their types, descriptions, and bounding boxes. Only works for PDFs. - priority: If True, attempts to process the job with priority if the user has priority - processing budget available; by default, sync jobs are prioritized above async - jobs. + priority: Workers poll the priority queue ahead of the standard queue, so priority jobs + start sooner when there is queued work; sync jobs are prioritized above async + jobs by default. extra_headers: Send extra headers @@ -320,7 +321,7 @@ async def run_job( options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=EditRunJobResponse, + cast_to=AsyncEditResponse, ) diff --git a/src/reducto/resources/extract.py b/src/reducto/resources/extract.py index 699226c5..743f1705 100644 --- a/src/reducto/resources/extract.py +++ b/src/reducto/resources/extract.py @@ -26,8 +26,8 @@ from ..types.parse_options_param import ParseOptionsParam from ..types.extract_run_response import ExtractRunResponse from ..types.async_config_v3_param import AsyncConfigV3Param -from ..types.async_extract_response import AsyncExtractResponse from ..types.extract_settings_param import ExtractSettingsParam +from ..types.shared.async_extract_response import AsyncExtractResponse __all__ = ["ExtractResource", "AsyncExtractResource"] diff --git a/src/reducto/resources/parse.py b/src/reducto/resources/parse.py index 5beacb07..732c6d24 100644 --- a/src/reducto/resources/parse.py +++ b/src/reducto/resources/parse.py @@ -28,8 +28,8 @@ from ..types.formatting_param import FormattingParam from ..types.spreadsheet_param import SpreadsheetParam from ..types.parse_run_response import ParseRunResponse -from ..types.async_parse_response import AsyncParseResponse from ..types.async_config_v3_param import AsyncConfigV3Param +from ..types.shared.async_parse_response import AsyncParseResponse __all__ = ["ParseResource", "AsyncParseResource"] @@ -102,7 +102,7 @@ def run( async_: AsyncConfigV3Param | Omit = omit, enhance: EnhanceParam | Omit = omit, formatting: FormattingParam | Omit = omit, - queue_priority: Literal["auto", "batch"] | Omit = omit, + queue_priority: Literal["auto", "standard", "batch"] | Omit = omit, retrieval: RetrievalParam | Omit = omit, settings: SettingsParam | Omit = omit, spreadsheet: SpreadsheetParam | Omit = omit, @@ -129,7 +129,7 @@ def run( async_: The configuration options for asynchronous processing (default synchronous). queue_priority: Queue priority. 'batch' for non-urgent work that processes when spare GPU - capacity is available. + capacity is available. 'auto' (alias: 'standard') uses the default queue. extra_headers: Send extra headers @@ -152,7 +152,7 @@ def run( settings: SettingsParam | Omit = omit, spreadsheet: SpreadsheetParam | Omit = omit, async_: AsyncConfigV3Param | Omit = omit, - queue_priority: Literal["auto", "batch"] | Omit = omit, + queue_priority: Literal["auto", "standard", "batch"] | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -191,7 +191,7 @@ def run_job( async_: AsyncConfigV3Param | Omit = omit, enhance: EnhanceParam | Omit = omit, formatting: FormattingParam | Omit = omit, - queue_priority: Literal["auto", "batch"] | Omit = omit, + queue_priority: Literal["auto", "standard", "batch"] | Omit = omit, retrieval: RetrievalParam | Omit = omit, settings: SettingsParam | Omit = omit, spreadsheet: SpreadsheetParam | Omit = omit, @@ -218,7 +218,7 @@ def run_job( async_: The configuration options for asynchronous processing (default synchronous). queue_priority: Queue priority. 'batch' for non-urgent work that processes when spare GPU - capacity is available. + capacity is available. 'auto' (alias: 'standard') uses the default queue. extra_headers: Send extra headers @@ -318,7 +318,7 @@ async def run( async_: AsyncConfigV3Param | Omit = omit, enhance: EnhanceParam | Omit = omit, formatting: FormattingParam | Omit = omit, - queue_priority: Literal["auto", "batch"] | Omit = omit, + queue_priority: Literal["auto", "standard", "batch"] | Omit = omit, retrieval: RetrievalParam | Omit = omit, settings: SettingsParam | Omit = omit, spreadsheet: SpreadsheetParam | Omit = omit, @@ -345,7 +345,7 @@ async def run( async_: The configuration options for asynchronous processing (default synchronous). queue_priority: Queue priority. 'batch' for non-urgent work that processes when spare GPU - capacity is available. + capacity is available. 'auto' (alias: 'standard') uses the default queue. extra_headers: Send extra headers @@ -368,7 +368,7 @@ async def run( settings: SettingsParam | Omit = omit, spreadsheet: SpreadsheetParam | Omit = omit, async_: AsyncConfigV3Param | Omit = omit, - queue_priority: Literal["auto", "batch"] | Omit = omit, + queue_priority: Literal["auto", "standard", "batch"] | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. extra_headers: Headers | None = None, @@ -407,7 +407,7 @@ async def run_job( async_: AsyncConfigV3Param | Omit = omit, enhance: EnhanceParam | Omit = omit, formatting: FormattingParam | Omit = omit, - queue_priority: Literal["auto", "batch"] | Omit = omit, + queue_priority: Literal["auto", "standard", "batch"] | Omit = omit, retrieval: RetrievalParam | Omit = omit, settings: SettingsParam | Omit = omit, spreadsheet: SpreadsheetParam | Omit = omit, @@ -434,7 +434,7 @@ async def run_job( async_: The configuration options for asynchronous processing (default synchronous). queue_priority: Queue priority. 'batch' for non-urgent work that processes when spare GPU - capacity is available. + capacity is available. 'auto' (alias: 'standard') uses the default queue. extra_headers: Send extra headers diff --git a/src/reducto/resources/pipeline.py b/src/reducto/resources/pipeline.py index 025334d8..a8b69071 100644 --- a/src/reducto/resources/pipeline.py +++ b/src/reducto/resources/pipeline.py @@ -2,6 +2,8 @@ from __future__ import annotations +import typing_extensions + import httpx from ..types import pipeline_run_params, pipeline_run_job_params @@ -16,10 +18,10 @@ async_to_streamed_response_wrapper, ) from .._base_client import make_request_options -from ..types.pipeline_response import PipelineResponse from ..types.async_config_v3_param import AsyncConfigV3Param from ..types.pipeline_settings_param import PipelineSettingsParam -from ..types.pipeline_run_job_response import PipelineRunJobResponse +from ..types.shared.pipeline_response import PipelineResponse +from ..types.shared.async_pipeline_response import AsyncPipelineResponse __all__ = ["PipelineResource", "AsyncPipelineResource"] @@ -44,6 +46,7 @@ def with_streaming_response(self) -> PipelineResourceWithStreamingResponse: """ return PipelineResourceWithStreamingResponse(self) + @typing_extensions.deprecated("deprecated") def run( self, *, @@ -98,6 +101,7 @@ def run( cast_to=PipelineResponse, ) + @typing_extensions.deprecated("deprecated") def run_job( self, *, @@ -111,7 +115,7 @@ def run_job( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> PipelineRunJobResponse: + ) -> AsyncPipelineResponse: """ Pipeline Async @@ -153,7 +157,7 @@ def run_job( options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=PipelineRunJobResponse, + cast_to=AsyncPipelineResponse, ) @@ -177,6 +181,7 @@ def with_streaming_response(self) -> AsyncPipelineResourceWithStreamingResponse: """ return AsyncPipelineResourceWithStreamingResponse(self) + @typing_extensions.deprecated("deprecated") async def run( self, *, @@ -231,6 +236,7 @@ async def run( cast_to=PipelineResponse, ) + @typing_extensions.deprecated("deprecated") async def run_job( self, *, @@ -244,7 +250,7 @@ async def run_job( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> PipelineRunJobResponse: + ) -> AsyncPipelineResponse: """ Pipeline Async @@ -286,7 +292,7 @@ async def run_job( options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=PipelineRunJobResponse, + cast_to=AsyncPipelineResponse, ) @@ -294,11 +300,15 @@ class PipelineResourceWithRawResponse: def __init__(self, pipeline: PipelineResource) -> None: self._pipeline = pipeline - self.run = to_raw_response_wrapper( - pipeline.run, + self.run = ( # pyright: ignore[reportDeprecated] + to_raw_response_wrapper( + pipeline.run, # pyright: ignore[reportDeprecated], + ) ) - self.run_job = to_raw_response_wrapper( - pipeline.run_job, + self.run_job = ( # pyright: ignore[reportDeprecated] + to_raw_response_wrapper( + pipeline.run_job, # pyright: ignore[reportDeprecated], + ) ) @@ -306,11 +316,15 @@ class AsyncPipelineResourceWithRawResponse: def __init__(self, pipeline: AsyncPipelineResource) -> None: self._pipeline = pipeline - self.run = async_to_raw_response_wrapper( - pipeline.run, + self.run = ( # pyright: ignore[reportDeprecated] + async_to_raw_response_wrapper( + pipeline.run, # pyright: ignore[reportDeprecated], + ) ) - self.run_job = async_to_raw_response_wrapper( - pipeline.run_job, + self.run_job = ( # pyright: ignore[reportDeprecated] + async_to_raw_response_wrapper( + pipeline.run_job, # pyright: ignore[reportDeprecated], + ) ) @@ -318,11 +332,15 @@ class PipelineResourceWithStreamingResponse: def __init__(self, pipeline: PipelineResource) -> None: self._pipeline = pipeline - self.run = to_streamed_response_wrapper( - pipeline.run, + self.run = ( # pyright: ignore[reportDeprecated] + to_streamed_response_wrapper( + pipeline.run, # pyright: ignore[reportDeprecated], + ) ) - self.run_job = to_streamed_response_wrapper( - pipeline.run_job, + self.run_job = ( # pyright: ignore[reportDeprecated] + to_streamed_response_wrapper( + pipeline.run_job, # pyright: ignore[reportDeprecated], + ) ) @@ -330,9 +348,13 @@ class AsyncPipelineResourceWithStreamingResponse: def __init__(self, pipeline: AsyncPipelineResource) -> None: self._pipeline = pipeline - self.run = async_to_streamed_response_wrapper( - pipeline.run, + self.run = ( # pyright: ignore[reportDeprecated] + async_to_streamed_response_wrapper( + pipeline.run, # pyright: ignore[reportDeprecated], + ) ) - self.run_job = async_to_streamed_response_wrapper( - pipeline.run_job, + self.run_job = ( # pyright: ignore[reportDeprecated] + async_to_streamed_response_wrapper( + pipeline.run_job, # pyright: ignore[reportDeprecated], + ) ) diff --git a/src/reducto/resources/split.py b/src/reducto/resources/split.py index ae2bcab7..91ca6c2a 100644 --- a/src/reducto/resources/split.py +++ b/src/reducto/resources/split.py @@ -18,12 +18,11 @@ async_to_streamed_response_wrapper, ) from .._base_client import make_request_options -from ..types.split_response import SplitResponse from ..types.parse_options_param import ParseOptionsParam from ..types.split_category_param import SplitCategoryParam from ..types.async_config_v3_param import AsyncConfigV3Param -from ..types.split_run_job_response import SplitRunJobResponse -from ..types.split_table_options_param import SplitTableOptionsParam +from ..types.shared.split_response import SplitResponse +from ..types.shared.async_split_response import AsyncSplitResponse __all__ = ["SplitResource", "AsyncSplitResource"] @@ -54,7 +53,7 @@ def run( input: split_run_params.Input, split_description: Iterable[SplitCategoryParam], parsing: ParseOptionsParam | Omit = omit, - settings: SplitTableOptionsParam | Omit = omit, + settings: split_run_params.Settings | Omit = omit, split_rules: str | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -118,7 +117,7 @@ def run_job( split_description: Iterable[SplitCategoryParam], async_: AsyncConfigV3Param | Omit = omit, parsing: ParseOptionsParam | Omit = omit, - settings: SplitTableOptionsParam | Omit = omit, + settings: split_run_job_params.Settings | Omit = omit, split_rules: str | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -126,7 +125,7 @@ def run_job( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> SplitRunJobResponse: + ) -> AsyncSplitResponse: """ Split Async @@ -175,7 +174,7 @@ def run_job( options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=SplitRunJobResponse, + cast_to=AsyncSplitResponse, ) @@ -205,7 +204,7 @@ async def run( input: split_run_params.Input, split_description: Iterable[SplitCategoryParam], parsing: ParseOptionsParam | Omit = omit, - settings: SplitTableOptionsParam | Omit = omit, + settings: split_run_params.Settings | Omit = omit, split_rules: str | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -269,7 +268,7 @@ async def run_job( split_description: Iterable[SplitCategoryParam], async_: AsyncConfigV3Param | Omit = omit, parsing: ParseOptionsParam | Omit = omit, - settings: SplitTableOptionsParam | Omit = omit, + settings: split_run_job_params.Settings | Omit = omit, split_rules: str | Omit = omit, # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs. # The extra values given here take precedence over values defined on the client or passed to this method. @@ -277,7 +276,7 @@ async def run_job( extra_query: Query | None = None, extra_body: Body | None = None, timeout: float | httpx.Timeout | None | NotGiven = not_given, - ) -> SplitRunJobResponse: + ) -> AsyncSplitResponse: """ Split Async @@ -326,7 +325,7 @@ async def run_job( options=make_request_options( extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout ), - cast_to=SplitRunJobResponse, + cast_to=AsyncSplitResponse, ) diff --git a/src/reducto/types/__init__.py b/src/reducto/types/__init__.py index 9994a7aa..a4a5b51d 100644 --- a/src/reducto/types/__init__.py +++ b/src/reducto/types/__init__.py @@ -2,28 +2,43 @@ from __future__ import annotations -from .shared import Upload as Upload +from .shared import ( + Upload as Upload, + Chunking as Chunking, + PageRange as PageRange, + TextAgentic as TextAgentic, + EditResponse as EditResponse, + TableAgentic as TableAgentic, + FigureAgentic as FigureAgentic, + ParseResponse as ParseResponse, + SplitResponse as SplitResponse, + ExtractResponse as ExtractResponse, + ClassifyResponse as ClassifyResponse, + PipelineResponse as PipelineResponse, + SplitLargeTables as SplitLargeTables, + WebhookConfigNew as WebhookConfigNew, + AsyncEditResponse as AsyncEditResponse, + SvixWebhookConfig as SvixWebhookConfig, + AsyncParseResponse as AsyncParseResponse, + AsyncSplitResponse as AsyncSplitResponse, + DirectWebhookConfig as DirectWebhookConfig, + AsyncExtractResponse as AsyncExtractResponse, + AsyncPipelineResponse as AsyncPipelineResponse, +) from .v3_extract import V3Extract as V3Extract from .edit_widget import EditWidget as EditWidget from .parse_usage import ParseUsage as ParseUsage from .bounding_box import BoundingBox as BoundingBox -from .edit_response import EditResponse as EditResponse from .enhance_param import EnhanceParam as EnhanceParam from .extract_usage import ExtractUsage as ExtractUsage -from .parse_response import ParseResponse as ParseResponse from .settings_param import SettingsParam as SettingsParam -from .split_response import SplitResponse as SplitResponse from .edit_run_params import EditRunParams as EditRunParams from .retrieval_param import RetrievalParam as RetrievalParam -from .extract_response import ExtractResponse as ExtractResponse from .formatting_param import FormattingParam as FormattingParam from .job_get_response import JobGetResponse as JobGetResponse -from .page_range_param import PageRangeParam as PageRangeParam from .parse_run_params import ParseRunParams as ParseRunParams from .split_run_params import SplitRunParams as SplitRunParams -from .classify_response import ClassifyResponse as ClassifyResponse from .edit_widget_param import EditWidgetParam as EditWidgetParam -from .pipeline_response import PipelineResponse as PipelineResponse from .spreadsheet_param import SpreadsheetParam as SpreadsheetParam from .bounding_box_param import BoundingBoxParam as BoundingBoxParam from .edit_options_param import EditOptionsParam as EditOptionsParam @@ -36,7 +51,6 @@ from .parse_options_param import ParseOptionsParam as ParseOptionsParam from .pipeline_run_params import PipelineRunParams as PipelineRunParams from .api_version_response import APIVersionResponse as APIVersionResponse -from .async_parse_response import AsyncParseResponse as AsyncParseResponse from .client_upload_params import ClientUploadParams as ClientUploadParams from .extract_run_response import ExtractRunResponse as ExtractRunResponse from .job_get_all_response import JobGetAllResponse as JobGetAllResponse @@ -45,15 +59,10 @@ from .split_run_job_params import SplitRunJobParams as SplitRunJobParams from .webhook_run_response import WebhookRunResponse as WebhookRunResponse from .async_config_v3_param import AsyncConfigV3Param as AsyncConfigV3Param -from .edit_run_job_response import EditRunJobResponse as EditRunJobResponse -from .async_extract_response import AsyncExtractResponse as AsyncExtractResponse from .extract_run_job_params import ExtractRunJobParams as ExtractRunJobParams from .extract_settings_param import ExtractSettingsParam as ExtractSettingsParam -from .split_run_job_response import SplitRunJobResponse as SplitRunJobResponse from .pipeline_run_job_params import PipelineRunJobParams as PipelineRunJobParams from .pipeline_settings_param import PipelineSettingsParam as PipelineSettingsParam from .async_parse_config_param import AsyncParseConfigParam as AsyncParseConfigParam from .deep_split_page_evidence import DeepSplitPageEvidence as DeepSplitPageEvidence -from .pipeline_run_job_response import PipelineRunJobResponse as PipelineRunJobResponse -from .split_table_options_param import SplitTableOptionsParam as SplitTableOptionsParam from .async_extract_config_param import AsyncExtractConfigParam as AsyncExtractConfigParam diff --git a/src/reducto/types/async_config_v3_param.py b/src/reducto/types/async_config_v3_param.py index 8d2e2156..9df1fde7 100644 --- a/src/reducto/types/async_config_v3_param.py +++ b/src/reducto/types/async_config_v3_param.py @@ -3,30 +3,14 @@ from __future__ import annotations from typing import Union, Optional -from typing_extensions import Literal, Required, TypeAlias, TypedDict +from typing_extensions import TypeAlias, TypedDict -from .._types import SequenceNotStr +from .shared_params.svix_webhook_config import SvixWebhookConfig +from .shared_params.direct_webhook_config import DirectWebhookConfig -__all__ = ["AsyncConfigV3Param", "Webhook", "WebhookSvixWebhookConfig", "WebhookDirectWebhookConfig"] +__all__ = ["AsyncConfigV3Param", "Webhook"] - -class WebhookSvixWebhookConfig(TypedDict, total=False): - channels: SequenceNotStr[str] - """ - A list of Svix channels the message will be delivered down, omit to send to all - channels. - """ - - mode: Literal["svix"] - - -class WebhookDirectWebhookConfig(TypedDict, total=False): - url: Required[str] - - mode: Literal["direct"] - - -Webhook: TypeAlias = Union[WebhookSvixWebhookConfig, WebhookDirectWebhookConfig] +Webhook: TypeAlias = Union[SvixWebhookConfig, DirectWebhookConfig] class AsyncConfigV3Param(TypedDict, total=False): @@ -35,9 +19,9 @@ class AsyncConfigV3Param(TypedDict, total=False): priority: bool """ - If True, attempts to process the job with priority if the user has priority - processing budget available; by default, sync jobs are prioritized above async - jobs. + Workers poll the priority queue ahead of the standard queue, so priority jobs + start sooner when there is queued work; sync jobs are prioritized above async + jobs by default. """ webhook: Optional[Webhook] diff --git a/src/reducto/types/async_parse_config_param.py b/src/reducto/types/async_parse_config_param.py index b20e2279..d0e4635b 100644 --- a/src/reducto/types/async_parse_config_param.py +++ b/src/reducto/types/async_parse_config_param.py @@ -44,10 +44,11 @@ class AsyncParseConfigParam(_AsyncParseConfigParamReservedKeywords, total=False) formatting: FormattingParam - queue_priority: Literal["auto", "batch"] + queue_priority: Literal["auto", "standard", "batch"] """Queue priority. 'batch' for non-urgent work that processes when spare GPU capacity is available. + 'auto' (alias: 'standard') uses the default queue. """ retrieval: RetrievalParam diff --git a/src/reducto/types/classify_response.py b/src/reducto/types/classify_response.py deleted file mode 100644 index 05d58978..00000000 --- a/src/reducto/types/classify_response.py +++ /dev/null @@ -1,56 +0,0 @@ -# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. - -from typing import List, Optional -from typing_extensions import Literal - -from .._models import BaseModel - -__all__ = [ - "ClassifyResponse", - "Result", - "ResponseConfidence", - "ResponseConfidenceCategory", - "ResponseConfidenceCategoryCriteriaConfidence", -] - - -class Result(BaseModel): - category: str - - -class ResponseConfidenceCategoryCriteriaConfidence(BaseModel): - """Confidence result for a single criterion.""" - - confidence: Literal["high", "low"] - - criterion: str - - -class ResponseConfidenceCategory(BaseModel): - """Confidence result for a category.""" - - category: str - - confidence: float - - criteria_confidence: List[ResponseConfidenceCategoryCriteriaConfidence] - - -class ResponseConfidence(BaseModel): - """Overall confidence breakdown for classification response.""" - - categories: List[ResponseConfidenceCategory] - - -class ClassifyResponse(BaseModel): - """Response from classify job - returned when polling /job/{job_id}""" - - job_id: str - - result: Result - - duration: Optional[float] = None - """The duration of the classify request in seconds.""" - - response_confidence: Optional[ResponseConfidence] = None - """Overall confidence breakdown for classification response.""" diff --git a/src/reducto/types/classify_run_params.py b/src/reducto/types/classify_run_params.py index 5c3db299..30780836 100644 --- a/src/reducto/types/classify_run_params.py +++ b/src/reducto/types/classify_run_params.py @@ -6,7 +6,7 @@ from typing_extensions import Required, TypeAlias, TypedDict from .._types import SequenceNotStr -from .page_range_param import PageRangeParam +from .shared_params import page_range from .shared_params.upload import Upload __all__ = ["ClassifyRunParams", "Input", "ClassificationSchema", "PageRange"] @@ -14,15 +14,12 @@ class ClassifyRunParams(TypedDict, total=False): input: Required[Input] - """For parse/split/extract pipelines, the URL of the document to be processed. + """The URL of the document to be classified. You can provide one of the following: - You can provide one of the following: 1. A publicly available URL 2. A presigned - S3 URL 3. A reducto:// prefixed URL obtained from the /upload endpoint after - directly uploading a document 4. A jobid:// prefixed URL obtained from a - previous /parse invocation 5. A list of URLs (for multi-document pipelines, V3 - API only) - - For edit pipelines, this should be a string containing the edit instructions + 1. A publicly available URL + 2. A presigned S3 URL + 3. A reducto:// prefixed URL obtained from the /upload endpoint after directly + uploading a document """ classification_schema: Iterable[ClassificationSchema] @@ -31,16 +28,22 @@ class ClassifyRunParams(TypedDict, total=False): document_metadata: Optional[str] """Optional document-level metadata to include in classification prompts.""" + force_url_result: bool + """Force the endpoint result to be returned in URL form.""" + page_range: Optional[PageRange] """The page range to process (1-indexed). - By default, the first 5 pages are used. If more than 25 pages are selected, only - the first 25 (after sorting) are used. Only applies to PDFs; ignored for other - document types. + By default, the first 5 pages are used. At most 10 pages can be selected. Only + applies to PDFs; ignored for other document types. """ - persist_results: bool - """If True, persist the results indefinitely. Defaults to False.""" + priority: bool + """ + Workers poll the priority queue ahead of the standard queue, so priority jobs + start sooner when there is queued work; sync jobs are prioritized above async + jobs by default. + """ Input: TypeAlias = Union[str, SequenceNotStr[str], Upload] @@ -63,4 +66,4 @@ class ClassificationSchema(TypedDict, total=False): """ -PageRange: TypeAlias = Union[PageRangeParam, Iterable[PageRangeParam], Iterable[int]] +PageRange: TypeAlias = Union[page_range.PageRange, Iterable[page_range.PageRange], Iterable[int]] diff --git a/src/reducto/types/client_upload_params.py b/src/reducto/types/client_upload_params.py index 60f0acc0..c8d7c223 100644 --- a/src/reducto/types/client_upload_params.py +++ b/src/reducto/types/client_upload_params.py @@ -5,10 +5,12 @@ from typing import Optional from typing_extensions import TypedDict +from .._types import FileTypes + __all__ = ["ClientUploadParams"] class ClientUploadParams(TypedDict, total=False): extension: Optional[str] - file: Optional[str] + file: Optional[FileTypes] diff --git a/src/reducto/types/edit_run_job_params.py b/src/reducto/types/edit_run_job_params.py index 5e3f628c..30a1c9e9 100644 --- a/src/reducto/types/edit_run_job_params.py +++ b/src/reducto/types/edit_run_job_params.py @@ -3,14 +3,14 @@ from __future__ import annotations from typing import Union, Iterable, Optional -from typing_extensions import Literal, Required, TypeAlias, TypedDict +from typing_extensions import Required, TypeAlias, TypedDict -from .._types import SequenceNotStr from .edit_widget_param import EditWidgetParam from .edit_options_param import EditOptionsParam from .shared_params.upload import Upload +from .shared_params.webhook_config_new import WebhookConfigNew -__all__ = ["EditRunJobParams", "DocumentURL", "Webhook"] +__all__ = ["EditRunJobParams", "DocumentURL"] class EditRunJobParams(TypedDict, total=False): @@ -37,32 +37,12 @@ class EditRunJobParams(TypedDict, total=False): priority: bool """ - If True, attempts to process the job with priority if the user has priority - processing budget available; by default, sync jobs are prioritized above async - jobs. + Workers poll the priority queue ahead of the standard queue, so priority jobs + start sooner when there is queued work; sync jobs are prioritized above async + jobs by default. """ - webhook: Webhook + webhook: WebhookConfigNew DocumentURL: TypeAlias = Union[str, Upload] - - -class Webhook(TypedDict, total=False): - channels: SequenceNotStr[str] - """ - A list of Svix channels the message will be delivered down, omit to send to all - channels. - """ - - metadata: object - """JSON metadata included in webhook request body""" - - mode: Literal["disabled", "svix", "direct"] - """The mode to use for webhook delivery. - - Defaults to 'disabled'. We recommend using 'svix' for production environments. - """ - - url: str - """The URL to send the webhook to (if using direct webhoook).""" diff --git a/src/reducto/types/edit_run_params.py b/src/reducto/types/edit_run_params.py index dd692c2f..027373cf 100644 --- a/src/reducto/types/edit_run_params.py +++ b/src/reducto/types/edit_run_params.py @@ -36,9 +36,9 @@ class EditRunParams(TypedDict, total=False): priority: bool """ - If True, attempts to process the job with priority if the user has priority - processing budget available; by default, sync jobs are prioritized above async - jobs. + Workers poll the priority queue ahead of the standard queue, so priority jobs + start sooner when there is queued work; sync jobs are prioritized above async + jobs by default. """ diff --git a/src/reducto/types/enhance_param.py b/src/reducto/types/enhance_param.py index 85d5569c..3597bf85 100644 --- a/src/reducto/types/enhance_param.py +++ b/src/reducto/types/enhance_param.py @@ -2,64 +2,44 @@ from __future__ import annotations -from typing import Union, Iterable, Optional -from typing_extensions import Literal, Required, TypeAlias, TypedDict +from typing import Union, Iterable +from typing_extensions import TypeAlias, TypedDict -__all__ = ["EnhanceParam", "Agentic", "AgenticTableAgentic", "AgenticFigureAgentic", "AgenticTextAgentic"] +from .shared_params.text_agentic import TextAgentic +from .shared_params.table_agentic import TableAgentic +from .shared_params.figure_agentic import FigureAgentic +__all__ = ["EnhanceParam", "Agentic"] -class AgenticTableAgentic(TypedDict, total=False): - scope: Required[Literal["table"]] +Agentic: TypeAlias = Union[TableAgentic, FigureAgentic, TextAgentic] - prompt: Optional[str] - """Custom prompt for table agentic.""" - - -class AgenticFigureAgentic(TypedDict, total=False): - scope: Required[Literal["figure"]] +class EnhanceParam(TypedDict, total=False): advanced_chart_agent: bool - """If True, use the advanced chart agent. Defaults to False.""" - - prompt: Optional[str] - """Custom prompt for figure agentic.""" - - return_overlays: bool - """If True, return overlays for the figure. - - This is so you can use the overlays to double check the quality of the - extraction """ - - -class AgenticTextAgentic(TypedDict, total=False): - scope: Required[Literal["text"]] - - prompt: Optional[str] - """Custom instructions for agentic text. - - Note: This only applies to form regions (key-value). + If True, run advanced chart extraction on figures classified as charts, without + requiring a figure-scoped agentic entry. Returns full structured series data + (chart_data) plus a reconstruction image re-drawn from that data. Higher + latency. Defaults to False. """ - -Agentic: TypeAlias = Union[AgenticTableAgentic, AgenticFigureAgentic, AgenticTextAgentic] - - -class EnhanceParam(TypedDict, total=False): agentic: Iterable[Agentic] """ - Agentic uses vision language models to enhance the accuracy of the output of - different types of extraction. This will incur a cost and latency increase. + For legacy Parse, agentic processing uses vision language models to improve + text, table, or figure extraction. With r-1, use agentic processing for custom + prompts or advanced chart extraction. Agentic processing adds latency. """ intelligent_ordering: bool """ - If True, use an advanced vision language model to improve reading order - accuracy, with a small increase in latency. Defaults to False. + For legacy Parse, if True, use an advanced vision language model to improve + reading order accuracy, with a small increase in latency. r-1 handles reading + order natively and ignores this setting. Defaults to False. """ summarize_figures: bool - """If True, summarize figures using a small vision language model. - + """ + For legacy Parse, if True, summarize figures using a separate vision language + model. r-1 generates figure descriptions natively and ignores this setting. Defaults to True. """ diff --git a/src/reducto/types/extract_response.py b/src/reducto/types/extract_response.py deleted file mode 100644 index 947aa58e..00000000 --- a/src/reducto/types/extract_response.py +++ /dev/null @@ -1,27 +0,0 @@ -# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. - -from typing import List, Optional - -from .._models import BaseModel -from .extract_usage import ExtractUsage - -__all__ = ["ExtractResponse"] - - -class ExtractResponse(BaseModel): - citations: Optional[List[object]] = None - """The citations corresponding to the extracted response.""" - - result: List[object] - """The extracted response in your provided schema. - - This is a list of dictionaries. If disable_chunking is True (default), then it - will be a list of length one. - """ - - usage: ExtractUsage - - job_id: Optional[str] = None - - studio_link: Optional[str] = None - """The link to the studio pipeline for the document.""" diff --git a/src/reducto/types/extract_run_response.py b/src/reducto/types/extract_run_response.py index e343fcfa..d19ea49a 100644 --- a/src/reducto/types/extract_run_response.py +++ b/src/reducto/types/extract_run_response.py @@ -4,7 +4,7 @@ from typing_extensions import TypeAlias from .v3_extract import V3Extract -from .async_extract_response import AsyncExtractResponse +from .shared.async_extract_response import AsyncExtractResponse __all__ = ["ExtractRunResponse"] diff --git a/src/reducto/types/extract_settings_param.py b/src/reducto/types/extract_settings_param.py index dd1ac60a..d802eed2 100644 --- a/src/reducto/types/extract_settings_param.py +++ b/src/reducto/types/extract_settings_param.py @@ -2,9 +2,13 @@ from __future__ import annotations -from typing_extensions import TypedDict +from typing import Union, Iterable, Optional +from typing_extensions import Literal, TypeAlias, TypedDict -__all__ = ["ExtractSettingsParam", "Citations"] +from .._types import SequenceNotStr +from .shared_params import page_range + +__all__ = ["ExtractSettingsParam", "Citations", "PageRange"] class Citations(TypedDict, total=False): @@ -16,10 +20,27 @@ class Citations(TypedDict, total=False): numerical_confidence: bool """If True, enable numeric citation confidence scores. Defaults to True.""" + parent_block: Literal["full", "bbox_only"] + """How much of the source parse block to embed on each citation's parentBlock. + + 'full' (default) embeds the verbatim source-block HTML in parentBlock.content. + 'bbox_only' suppresses parentBlock.content (returned as an empty string) while + keeping parentBlock.bbox and all citation-level fields — this can drastically + shrink responses on table-heavy schemas where the same source block is cited + many times. + """ + + +PageRange: TypeAlias = Union[page_range.PageRange, Iterable[page_range.PageRange], Iterable[int], SequenceNotStr[str]] + class ExtractSettingsParam(TypedDict, total=False): array_extract: bool - """If True, use array extraction.""" + """ + Deprecated: prefer deep_extract, which supersedes array extraction for complex + and long (array-heavy) extractions via an agentic loop (at higher cost and + latency). If True, use array extraction. + """ citations: Citations """The citations to use for the extraction.""" @@ -31,6 +52,9 @@ class ExtractSettingsParam(TypedDict, total=False): accuracy is critical. """ + force_url_result: bool + """Force the endpoint result to be returned in URL form.""" + include_images: bool """If True, include images in the extraction.""" @@ -39,3 +63,10 @@ class ExtractSettingsParam(TypedDict, total=False): If True, jobs will be processed with a higher throughput and priority at a higher cost. Defaults to False. """ + + page_range: Optional[PageRange] + """The page range to extract from (1-indexed). + + By default, the entire document is used. For spreadsheets, you can also provide + a list of sheet names. + """ diff --git a/src/reducto/types/extract_usage.py b/src/reducto/types/extract_usage.py index 2d6a0a30..dd95042e 100644 --- a/src/reducto/types/extract_usage.py +++ b/src/reducto/types/extract_usage.py @@ -5,7 +5,29 @@ from .._models import BaseModel -__all__ = ["ExtractUsage"] +__all__ = ["ExtractUsage", "UsageBreakdown"] + + +class UsageBreakdown(BaseModel): + """Raw extract quantities for accounts on the new pricing model. + + ``extract_fields`` is reported but not billed at launch. The add-on + quantities (``ocr_pages``, ``charts``, ``prompted_blocks``) come from the + parse bundled into the extract job; its page cost is covered by + ``extract_pages`` but its add-ons are billed separately. + """ + + extract_model: Literal["Extract", "Deep Extract"] + + charts: Optional[int] = None + + extract_fields: Optional[int] = None + + extract_pages: Optional[int] = None + + ocr_pages: Optional[int] = None + + prompted_blocks: Optional[int] = None class ExtractUsage(BaseModel): @@ -16,3 +38,12 @@ class ExtractUsage(BaseModel): credits: Optional[float] = None extract_mode: Optional[Literal["super_agent", "extract", "spreadsheet_agent"]] = None + + usage_breakdown: Optional[UsageBreakdown] = None + """Raw extract quantities for accounts on the new pricing model. + + `extract_fields` is reported but not billed at launch. The add-on quantities + (`ocr_pages`, `charts`, `prompted_blocks`) come from the parse bundled into the + extract job; its page cost is covered by `extract_pages` but its add-ons are + billed separately. + """ diff --git a/src/reducto/types/formatting_param.py b/src/reducto/types/formatting_param.py index c583c0d0..619c14f7 100644 --- a/src/reducto/types/formatting_param.py +++ b/src/reducto/types/formatting_param.py @@ -16,7 +16,11 @@ class FormattingParam(TypedDict, total=False): """ include: List[Literal["change_tracking", "highlight", "comments", "hyperlinks", "signatures", "ignore_watermarks"]] - """A list of formatting to include in the output.""" + """For legacy Parse, the formatting details to include in the output. + + r-1 handles highlights, signatures, and watermarks natively and ignores those + values. r-1 does not support hyperlinks. + """ merge_tables: bool """ @@ -25,8 +29,8 @@ class FormattingParam(TypedDict, total=False): """ table_output_format: Literal["html", "json", "md", "jsonbbox", "dynamic", "csv"] - """The mode to use for table output. + """The table output format. Defaults to dynamic, which returns md for simpler tables and html for more - complex tables. + complex tables. r-1 does not support jsonbbox. """ diff --git a/src/reducto/types/job_get_all_response.py b/src/reducto/types/job_get_all_response.py index d7a1a7ee..591a8a18 100644 --- a/src/reducto/types/job_get_all_response.py +++ b/src/reducto/types/job_get_all_response.py @@ -24,7 +24,7 @@ class Job(BaseModel): total_pages: Optional[int] = None - type: Literal["Parse", "Extract", "Split", "Edit", "Pipeline", "Classify"] + type: Literal["Parse", "Extract", "Split", "Edit", "Pipeline", "Classify", "Chart"] bucket: Optional[object] = None diff --git a/src/reducto/types/job_get_response.py b/src/reducto/types/job_get_response.py index 5190ae91..bdb4768a 100644 --- a/src/reducto/types/job_get_response.py +++ b/src/reducto/types/job_get_response.py @@ -1,34 +1,155 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. -from typing import Union, Optional +from typing import Dict, Union, Optional from datetime import datetime -from typing_extensions import Literal, TypeAlias +from typing_extensions import Literal, Annotated, TypeAlias +from .._utils import PropertyInfo from .._models import BaseModel from .v3_extract import V3Extract -from .edit_response import EditResponse -from .parse_response import ParseResponse -from .split_response import SplitResponse -from .extract_response import ExtractResponse -from .classify_response import ClassifyResponse -from .pipeline_response import PipelineResponse +from .shared.edit_response import EditResponse +from .shared.parse_response import ParseResponse +from .shared.split_response import SplitResponse +from .shared.extract_response import ExtractResponse +from .shared.classify_response import ClassifyResponse +from .shared.pipeline_response import PipelineResponse __all__ = [ "JobGetResponse", "AsyncJobResponse", + "AsyncJobResponseError", "AsyncJobResponseResult", + "AsyncJobResponseResultChartResponse", + "AsyncJobResponseResultChartResponseResult", + "AsyncJobResponseResultChartResponseUsage", "EnhancedAsyncJobResponse", + "EnhancedAsyncJobResponseError", "EnhancedAsyncJobResponseResult", + "EnhancedAsyncJobResponseResultChartResponse", + "EnhancedAsyncJobResponseResultChartResponseResult", + "EnhancedAsyncJobResponseResultChartResponseUsage", ] -AsyncJobResponseResult: TypeAlias = Union[ - ParseResponse, ExtractResponse, SplitResponse, EditResponse, PipelineResponse, V3Extract, ClassifyResponse, None + +class AsyncJobResponseError(BaseModel): + """Structured error body returned to customers. + + Matches the format specified in ``ERROR_POLICY.md``. + """ + + code: int + + message: str + + name: Literal[ + "TIMEOUT", + "CAPACITY_TIMEOUT", + "CUSTOMER_TIMEOUT", + "INTERNAL_ERROR", + "SERVICE_UNAVAILABLE", + "GPU_ALLOCATION_ERROR", + "GPU_POOL_SATURATED", + "BATCH_QUEUE_FULL", + "JOB_STATE_ERROR", + "DOCUMENT_CORRUPT", + "DOCUMENT_EMPTY", + "DOCUMENT_UNSUPPORTED", + "DOCUMENT_TOO_LARGE", + "OFFICE_CONVERSION_TOO_LARGE", + "IMAGE_TOO_LARGE", + "IMAGE_TOO_SMALL", + "IMAGE_INVALID_ASPECT_RATIO", + "DOCUMENT_PASSWORD_PROTECTED", + "FORM_FILL_FAILED", + "INTERNAL_INVARIANT_VIOLATION", + "GPU_UNAVAILABLE", + "CONTEXT_WINDOW_EXCEEDED", + "PROCESSING_FAILED", + "INFERENCE_METHOD_UNSUPPORTED", + "SUBPROCESS_CRASHED", + "BATCH_ORPHANED", + "OVERSIZED_RESULT", + "LLM_OUTPUT_PARSE_FAILED", + "LLM_PROVIDER_ERROR", + "INVALID_CONFIG", + "INVALID_SCHEMA", + "AUTH_ERROR", + "NOT_APPLICABLE", + "REGION_UNAVAILABLE", + "NOT_FOUND", + "JOB_DELETION_IN_PROGRESS", + "JOB_DELETED", + "JOB_NOT_COMPLETE", + "JOB_CANCELLED", + "RATE_LIMIT", + "CELL_COUNT_EXCEEDED", + "URL_NOT_ALLOWED", + ] + """Machine-readable error names returned in API error responses. + + Each member maps to a category (Transient / Processing / Input) and a default + HTTP status code defined in `ERROR_CODE_DEFAULTS`. The enum value is the string + customers see in the `error.name` field. + """ + + job_id: Optional[str] = None + + +class AsyncJobResponseResultChartResponseResult(BaseModel): + chart_data: Dict[str, object] + + reconstruction_url: str + + summary: str + + verified: bool + + +class AsyncJobResponseResultChartResponseUsage(BaseModel): + credits: Optional[float] = None + + num_charts: Optional[Literal[1]] = None + + +class AsyncJobResponseResultChartResponse(BaseModel): + """Response from synchronous and persisted chart extraction jobs.""" + + job_id: str + + result: AsyncJobResponseResultChartResponseResult + + duration: Optional[float] = None + + response_type: Optional[Literal["chart"]] = None + + usage: Optional[AsyncJobResponseResultChartResponseUsage] = None + + +AsyncJobResponseResult: TypeAlias = Annotated[ + Union[ + ParseResponse, + ExtractResponse, + SplitResponse, + EditResponse, + PipelineResponse, + V3Extract, + ClassifyResponse, + AsyncJobResponseResultChartResponse, + None, + ], + PropertyInfo(discriminator="response_type"), ] class AsyncJobResponse(BaseModel): status: Literal["Pending", "Completed", "Failed", "Idle"] + error: Optional[AsyncJobResponseError] = None + """Structured error body returned to customers. + + Matches the format specified in `ERROR_POLICY.md`. + """ + progress: Optional[float] = None reason: Optional[str] = None @@ -37,8 +158,113 @@ class AsyncJobResponse(BaseModel): """Response from classify job - returned when polling /job/{job_id}""" -EnhancedAsyncJobResponseResult: TypeAlias = Union[ - ParseResponse, ExtractResponse, SplitResponse, EditResponse, PipelineResponse, V3Extract, ClassifyResponse, None +class EnhancedAsyncJobResponseError(BaseModel): + """Structured error body returned to customers. + + Matches the format specified in ``ERROR_POLICY.md``. + """ + + code: int + + message: str + + name: Literal[ + "TIMEOUT", + "CAPACITY_TIMEOUT", + "CUSTOMER_TIMEOUT", + "INTERNAL_ERROR", + "SERVICE_UNAVAILABLE", + "GPU_ALLOCATION_ERROR", + "GPU_POOL_SATURATED", + "BATCH_QUEUE_FULL", + "JOB_STATE_ERROR", + "DOCUMENT_CORRUPT", + "DOCUMENT_EMPTY", + "DOCUMENT_UNSUPPORTED", + "DOCUMENT_TOO_LARGE", + "OFFICE_CONVERSION_TOO_LARGE", + "IMAGE_TOO_LARGE", + "IMAGE_TOO_SMALL", + "IMAGE_INVALID_ASPECT_RATIO", + "DOCUMENT_PASSWORD_PROTECTED", + "FORM_FILL_FAILED", + "INTERNAL_INVARIANT_VIOLATION", + "GPU_UNAVAILABLE", + "CONTEXT_WINDOW_EXCEEDED", + "PROCESSING_FAILED", + "INFERENCE_METHOD_UNSUPPORTED", + "SUBPROCESS_CRASHED", + "BATCH_ORPHANED", + "OVERSIZED_RESULT", + "LLM_OUTPUT_PARSE_FAILED", + "LLM_PROVIDER_ERROR", + "INVALID_CONFIG", + "INVALID_SCHEMA", + "AUTH_ERROR", + "NOT_APPLICABLE", + "REGION_UNAVAILABLE", + "NOT_FOUND", + "JOB_DELETION_IN_PROGRESS", + "JOB_DELETED", + "JOB_NOT_COMPLETE", + "JOB_CANCELLED", + "RATE_LIMIT", + "CELL_COUNT_EXCEEDED", + "URL_NOT_ALLOWED", + ] + """Machine-readable error names returned in API error responses. + + Each member maps to a category (Transient / Processing / Input) and a default + HTTP status code defined in `ERROR_CODE_DEFAULTS`. The enum value is the string + customers see in the `error.name` field. + """ + + job_id: Optional[str] = None + + +class EnhancedAsyncJobResponseResultChartResponseResult(BaseModel): + chart_data: Dict[str, object] + + reconstruction_url: str + + summary: str + + verified: bool + + +class EnhancedAsyncJobResponseResultChartResponseUsage(BaseModel): + credits: Optional[float] = None + + num_charts: Optional[Literal[1]] = None + + +class EnhancedAsyncJobResponseResultChartResponse(BaseModel): + """Response from synchronous and persisted chart extraction jobs.""" + + job_id: str + + result: EnhancedAsyncJobResponseResultChartResponseResult + + duration: Optional[float] = None + + response_type: Optional[Literal["chart"]] = None + + usage: Optional[EnhancedAsyncJobResponseResultChartResponseUsage] = None + + +EnhancedAsyncJobResponseResult: TypeAlias = Annotated[ + Union[ + ParseResponse, + ExtractResponse, + SplitResponse, + EditResponse, + PipelineResponse, + V3Extract, + ClassifyResponse, + EnhancedAsyncJobResponseResultChartResponse, + None, + ], + PropertyInfo(discriminator="response_type"), ] @@ -51,6 +277,12 @@ class EnhancedAsyncJobResponse(BaseModel): duration: Optional[float] = None + error: Optional[EnhancedAsyncJobResponseError] = None + """Structured error body returned to customers. + + Matches the format specified in `ERROR_POLICY.md`. + """ + num_pages: Optional[int] = None progress: Optional[float] = None @@ -66,7 +298,7 @@ class EnhancedAsyncJobResponse(BaseModel): total_pages: Optional[int] = None - type: Optional[Literal["Parse", "Extract", "Split", "Edit", "Pipeline", "Classify"]] = None + type: Optional[Literal["Parse", "Extract", "Split", "Edit", "Pipeline", "Classify", "Chart"]] = None JobGetResponse: TypeAlias = Union[AsyncJobResponse, EnhancedAsyncJobResponse] diff --git a/src/reducto/types/parse_run_job_params.py b/src/reducto/types/parse_run_job_params.py index fbcd1c76..85c0ca41 100644 --- a/src/reducto/types/parse_run_job_params.py +++ b/src/reducto/types/parse_run_job_params.py @@ -38,10 +38,11 @@ class ParseRunJobParams(TypedDict, total=False): formatting: FormattingParam - queue_priority: Literal["auto", "batch"] + queue_priority: Literal["auto", "standard", "batch"] """Queue priority. 'batch' for non-urgent work that processes when spare GPU capacity is available. + 'auto' (alias: 'standard') uses the default queue. """ retrieval: RetrievalParam diff --git a/src/reducto/types/parse_run_params.py b/src/reducto/types/parse_run_params.py index a929024a..704e50c7 100644 --- a/src/reducto/types/parse_run_params.py +++ b/src/reducto/types/parse_run_params.py @@ -65,10 +65,11 @@ class AsyncParseConfig(TypedDict, total=False): formatting: FormattingParam - queue_priority: Literal["auto", "batch"] + queue_priority: Literal["auto", "standard", "batch"] """Queue priority. 'batch' for non-urgent work that processes when spare GPU capacity is available. + 'auto' (alias: 'standard') uses the default queue. """ retrieval: RetrievalParam diff --git a/src/reducto/types/parse_run_response.py b/src/reducto/types/parse_run_response.py index f12989e7..6c78116a 100644 --- a/src/reducto/types/parse_run_response.py +++ b/src/reducto/types/parse_run_response.py @@ -3,8 +3,8 @@ from typing import Union from typing_extensions import TypeAlias -from .parse_response import ParseResponse -from .async_parse_response import AsyncParseResponse +from .shared.parse_response import ParseResponse +from .shared.async_parse_response import AsyncParseResponse __all__ = ["ParseRunResponse"] diff --git a/src/reducto/types/parse_usage.py b/src/reducto/types/parse_usage.py index 74abb42d..80066b6e 100644 --- a/src/reducto/types/parse_usage.py +++ b/src/reducto/types/parse_usage.py @@ -1,10 +1,82 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. -from typing import Dict, Optional +from typing import Dict, List, Union, Optional +from typing_extensions import Literal, TypeAlias from .._models import BaseModel -__all__ = ["ParseUsage"] +__all__ = [ + "ParseUsage", + "UsageBreakdown", + "UsageBreakdownParseUsageBreakdown", + "UsageBreakdownSplitUsageBreakdown", + "UsageBreakdownEditUsageBreakdown", +] + + +class UsageBreakdownParseUsageBreakdown(BaseModel): + """Raw parse quantities for accounts on the new (Q3 2026) pricing model. + + ``parse_model`` is "R-1" for the new parse model and "Legacy" for the + legacy parse pipeline. A legacy-pipeline parse carries its cost in + ``legacy_parse_credits``; add-on quantities (``ocr_pages``, ``charts``, + ``prompted_blocks``) apply to the new parse model only. + """ + + parse_model: Literal["R-1", "Legacy"] + + tier: Literal["Default", "Batch"] + + charts: Optional[int] = None + + legacy_parse_credits: Optional[float] = None + + ocr_pages: Optional[int] = None + + parse_native_pages: Optional[int] = None + + parse_pages: Optional[int] = None + + prompted_blocks: Optional[int] = None + + +class UsageBreakdownSplitUsageBreakdown(BaseModel): + """Raw split quantities for accounts on the new pricing model. + + The add-on quantities (``ocr_pages``, ``charts``, ``prompted_blocks``) + come from the parse bundled into the split job; its page cost is covered + by ``split_pages`` but its add-ons are billed separately. + """ + + split_model: Literal["Split", "Deep Split"] + + charts: Optional[int] = None + + ocr_pages: Optional[int] = None + + prompted_blocks: Optional[int] = None + + split_pages: Optional[int] = None + + +class UsageBreakdownEditUsageBreakdown(BaseModel): + """Raw edit quantities for accounts on the new pricing model. + + ``edit_pages`` is the page count billed at the ``edit_model`` rate. A job + with both normal and prefilled pages reports ``edit_model="Normal"`` with + the prefilled pages in ``prefill_pages``, billed at the "Prefill" rate. + """ + + edit_model: Literal["Normal", "Prefill"] + + edit_pages: Optional[int] = None + + prefill_pages: Optional[int] = None + + +UsageBreakdown: TypeAlias = Union[ + UsageBreakdownParseUsageBreakdown, UsageBreakdownSplitUsageBreakdown, UsageBreakdownEditUsageBreakdown, None +] class ParseUsage(BaseModel): @@ -13,3 +85,43 @@ class ParseUsage(BaseModel): credit_breakdown: Optional[Dict[str, float]] = None credits: Optional[float] = None + + non_empty_cell_count: Optional[int] = None + """Total non-empty cells across all sheets. Only set for spreadsheet inputs.""" + + page_billing_breakdown: Optional[ + Dict[ + str, + List[ + Literal[ + "page", + "html_page", + "docx_native_page", + "agentic", + "complex", + "chart_agent", + "spreadsheet_cells", + "billable_spreadsheet_pages", + "enrich_table", + "figure_summary", + "table_summary", + "key_value", + "agentic_text", + "promptable_agentic_text", + "reducto_lite_page", + ] + ], + ] + ] = None + """Per-page breakdown of features used. + + Maps 1-indexed page numbers (as strings) to the list of billing features applied + on that page (e.g. 'page', 'complex', 'chart_agent'). + """ + + usage_breakdown: Optional[UsageBreakdown] = None + """Raw usage quantities. + + Only set for accounts on the new pricing model; credit fields are omitted for + those accounts. + """ diff --git a/src/reducto/types/pipeline_run_job_response.py b/src/reducto/types/pipeline_run_job_response.py deleted file mode 100644 index efe6e839..00000000 --- a/src/reducto/types/pipeline_run_job_response.py +++ /dev/null @@ -1,9 +0,0 @@ -# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. - -from .._models import BaseModel - -__all__ = ["PipelineRunJobResponse"] - - -class PipelineRunJobResponse(BaseModel): - job_id: str diff --git a/src/reducto/types/retrieval_param.py b/src/reducto/types/retrieval_param.py index 3cf32952..cbff9cf6 100644 --- a/src/reducto/types/retrieval_param.py +++ b/src/reducto/types/retrieval_param.py @@ -2,31 +2,12 @@ from __future__ import annotations -from typing import List, Optional +from typing import List from typing_extensions import Literal, TypedDict -__all__ = ["RetrievalParam", "Chunking"] +from .shared_params.chunking import Chunking - -class Chunking(TypedDict, total=False): - chunk_mode: Literal["variable", "section", "page", "disabled", "block", "page_sections"] - """Choose how to partition chunks. - - Variable mode chunks by character length and visual context. Section mode chunks - by section headers. Page mode chunks according to pages. Page sections mode - chunks first by page, then by sections within each page. Disabled returns one - single chunk. - """ - - chunk_overlap: int - """Number of characters of overlap to include from adjacent chunks. Defaults to 0.""" - - chunk_size: Optional[int] - """ - The approximate size of chunks (in characters) that the document will be split - into. Defaults to null, in which case the chunk size is variable between 250 - - 1500 characters. - """ +__all__ = ["RetrievalParam"] class RetrievalParam(TypedDict, total=False): diff --git a/src/reducto/types/settings_param.py b/src/reducto/types/settings_param.py index efa50cad..6646c245 100644 --- a/src/reducto/types/settings_param.py +++ b/src/reducto/types/settings_param.py @@ -3,14 +3,44 @@ from __future__ import annotations from typing import List, Union, Iterable, Optional -from typing_extensions import Literal, TypeAlias, TypedDict +from typing_extensions import Literal, Required, TypeAlias, TypedDict from .._types import SequenceNotStr -from .page_range_param import PageRangeParam +from .shared_params import page_range -__all__ = ["SettingsParam", "PageRange"] +__all__ = ["SettingsParam", "HybridVpc", "PageRange", "TenantThrottling"] -PageRange: TypeAlias = Union[PageRangeParam, Iterable[PageRangeParam], Iterable[int], SequenceNotStr[str]] + +class HybridVpc(TypedDict, total=False): + """Hybrid VPC request-scoped settings.""" + + environment: Optional[str] + """Named Hybrid VPC environment to use for this request. + + Only applies when your organization has Hybrid VPC environments configured. + """ + + +PageRange: TypeAlias = Union[page_range.PageRange, Iterable[page_range.PageRange], Iterable[int], SequenceNotStr[str]] + + +class TenantThrottling(TypedDict, total=False): + """Per-tenant throttling for multi-tenant applications. + + Tag each request with your tenant's id to bound how much of your account's concurrency a single tenant can consume. Account-level throttles still apply. + """ + + tenant_id: Required[str] + """ + Your identifier for the tenant (customer, workspace, organization) this request + belongs to. Used only for noisy-neighbor throttling inside your account. + """ + + max_share: float + """ + Maximum fraction of your account's concurrency ceiling this tenant may use, + between 0 (exclusive) and 1. Defaults to 0.5. + """ class SettingsParam(TypedDict, total=False): @@ -20,11 +50,27 @@ class SettingsParam(TypedDict, total=False): embed_pdf_metadata: bool """If True, embed OCR metadata into the returned PDF. Defaults to False.""" + embed_pdf_metadata_dpi: int + """ + Render DPI used when rasterizing the source PDF before embedding the OCR text + layer (only applies when `embed_pdf_metadata` is True). Lower values produce + dramatically smaller output PDFs; higher values preserve more detail when zoomed + past 200%. Defaults to 100 (good for on-screen viewing); raise toward the source + scan DPI for crisper output. Min 50, max 250. + """ + + extract_document_properties: bool + """If True, return properties embedded in the original document. + + Defaults to False. + """ + extraction_mode: Literal["ocr", "hybrid"] - """The mode to use for text extraction from PDFs. + """The text extraction method for legacy Parse. - OCR mode uses optical character recognition only. Hybrid mode combines OCR with - embedded PDF text for best accuracy (default). + OCR uses optical character recognition only. Hybrid combines OCR with embedded + PDF text. r-1 uses native full-page processing and ignores this setting. + Defaults to hybrid. """ force_file_extension: Optional[str] @@ -33,11 +79,24 @@ class SettingsParam(TypedDict, total=False): force_url_result: bool """Force the result to be returned in URL form.""" + hybrid_vpc: HybridVpc + """Hybrid VPC request-scoped settings.""" + + model: Optional[Literal["r-1", "legacy"]] + """The parse model to use. + + 'r-1' is the R-1 full-page parse model, which parses each page in a single + generation. 'legacy' is the previous parsing pipeline. Defaults to 'legacy' + unless your organization was created on the r-1 plan, in which case it defaults + to 'r-1'. + """ + ocr_system: Literal["standard", "legacy"] - """Standard is our best multilingual OCR system. + """The OCR system for legacy Parse. - Legacy only supports germanic languages and is available for backwards - compatibility. + Standard is the best multilingual OCR system. Legacy supports Germanic languages + and remains available for backwards compatibility. r-1 uses native full-page + processing and ignores this setting. Defaults to standard. """ page_range: Optional[PageRange] @@ -59,5 +118,12 @@ class SettingsParam(TypedDict, total=False): return_ocr_data: bool """If True, return OCR data in the result. Defaults to False.""" + tenant_throttling: Optional[TenantThrottling] + """Per-tenant throttling for multi-tenant applications. + + Tag each request with your tenant's id to bound how much of your account's + concurrency a single tenant can consume. Account-level throttles still apply. + """ + timeout: Optional[float] """The timeout for the job in seconds.""" diff --git a/src/reducto/types/shared/__init__.py b/src/reducto/types/shared/__init__.py index e9bf9399..8ac14976 100644 --- a/src/reducto/types/shared/__init__.py +++ b/src/reducto/types/shared/__init__.py @@ -1,3 +1,23 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. from .upload import Upload as Upload +from .chunking import Chunking as Chunking +from .page_range import PageRange as PageRange +from .text_agentic import TextAgentic as TextAgentic +from .edit_response import EditResponse as EditResponse +from .table_agentic import TableAgentic as TableAgentic +from .figure_agentic import FigureAgentic as FigureAgentic +from .parse_response import ParseResponse as ParseResponse +from .split_response import SplitResponse as SplitResponse +from .extract_response import ExtractResponse as ExtractResponse +from .classify_response import ClassifyResponse as ClassifyResponse +from .pipeline_response import PipelineResponse as PipelineResponse +from .split_large_tables import SplitLargeTables as SplitLargeTables +from .webhook_config_new import WebhookConfigNew as WebhookConfigNew +from .async_edit_response import AsyncEditResponse as AsyncEditResponse +from .svix_webhook_config import SvixWebhookConfig as SvixWebhookConfig +from .async_parse_response import AsyncParseResponse as AsyncParseResponse +from .async_split_response import AsyncSplitResponse as AsyncSplitResponse +from .direct_webhook_config import DirectWebhookConfig as DirectWebhookConfig +from .async_extract_response import AsyncExtractResponse as AsyncExtractResponse +from .async_pipeline_response import AsyncPipelineResponse as AsyncPipelineResponse diff --git a/src/reducto/types/edit_run_job_response.py b/src/reducto/types/shared/async_edit_response.py similarity index 50% rename from src/reducto/types/edit_run_job_response.py rename to src/reducto/types/shared/async_edit_response.py index 967f46d8..da6fcba4 100644 --- a/src/reducto/types/edit_run_job_response.py +++ b/src/reducto/types/shared/async_edit_response.py @@ -1,9 +1,9 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. -from .._models import BaseModel +from ..._models import BaseModel -__all__ = ["EditRunJobResponse"] +__all__ = ["AsyncEditResponse"] -class EditRunJobResponse(BaseModel): +class AsyncEditResponse(BaseModel): job_id: str diff --git a/src/reducto/types/async_extract_response.py b/src/reducto/types/shared/async_extract_response.py similarity index 84% rename from src/reducto/types/async_extract_response.py rename to src/reducto/types/shared/async_extract_response.py index 7ee83afa..5bafe1ca 100644 --- a/src/reducto/types/async_extract_response.py +++ b/src/reducto/types/shared/async_extract_response.py @@ -1,6 +1,6 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. -from .._models import BaseModel +from ..._models import BaseModel __all__ = ["AsyncExtractResponse"] diff --git a/src/reducto/types/async_parse_response.py b/src/reducto/types/shared/async_parse_response.py similarity index 84% rename from src/reducto/types/async_parse_response.py rename to src/reducto/types/shared/async_parse_response.py index 41610236..9df7fe5d 100644 --- a/src/reducto/types/async_parse_response.py +++ b/src/reducto/types/shared/async_parse_response.py @@ -1,6 +1,6 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. -from .._models import BaseModel +from ..._models import BaseModel __all__ = ["AsyncParseResponse"] diff --git a/src/reducto/types/shared/async_pipeline_response.py b/src/reducto/types/shared/async_pipeline_response.py new file mode 100644 index 00000000..ca6a8829 --- /dev/null +++ b/src/reducto/types/shared/async_pipeline_response.py @@ -0,0 +1,9 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from ..._models import BaseModel + +__all__ = ["AsyncPipelineResponse"] + + +class AsyncPipelineResponse(BaseModel): + job_id: str diff --git a/src/reducto/types/split_run_job_response.py b/src/reducto/types/shared/async_split_response.py similarity index 50% rename from src/reducto/types/split_run_job_response.py rename to src/reducto/types/shared/async_split_response.py index 404b32fe..5294328b 100644 --- a/src/reducto/types/split_run_job_response.py +++ b/src/reducto/types/shared/async_split_response.py @@ -1,9 +1,9 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. -from .._models import BaseModel +from ..._models import BaseModel -__all__ = ["SplitRunJobResponse"] +__all__ = ["AsyncSplitResponse"] -class SplitRunJobResponse(BaseModel): +class AsyncSplitResponse(BaseModel): job_id: str diff --git a/src/reducto/types/shared/chunking.py b/src/reducto/types/shared/chunking.py new file mode 100644 index 00000000..b56c1250 --- /dev/null +++ b/src/reducto/types/shared/chunking.py @@ -0,0 +1,29 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional +from typing_extensions import Literal + +from ..._models import BaseModel + +__all__ = ["Chunking"] + + +class Chunking(BaseModel): + chunk_mode: Optional[Literal["variable", "section", "page", "disabled", "block", "page_sections"]] = None + """Choose how to partition chunks. + + Variable mode chunks by character length and visual context. Section mode chunks + by section headers. Page mode chunks according to pages. Page sections mode + chunks first by page, then by sections within each page. Disabled returns one + single chunk. + """ + + chunk_overlap: Optional[int] = None + """Number of characters of overlap to include from adjacent chunks. Defaults to 0.""" + + chunk_size: Optional[int] = None + """ + The approximate size of chunks (in characters) that the document will be split + into. Defaults to null, in which case the chunk size is variable between 250 - + 1500 characters. + """ diff --git a/src/reducto/types/shared/classify_response.py b/src/reducto/types/shared/classify_response.py new file mode 100644 index 00000000..2762702f --- /dev/null +++ b/src/reducto/types/shared/classify_response.py @@ -0,0 +1,103 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import List, Union, Optional +from typing_extensions import Literal, TypeAlias + +from ..._models import BaseModel + +__all__ = [ + "ClassifyResponse", + "Result", + "ResultClassifyResponseCategory", + "ResultURLResult", + "ResponseConfidence", + "ResponseConfidenceCategory", + "ResponseConfidenceCategoryCriteriaConfidence", + "Usage", + "UsageUsageBreakdown", +] + + +class ResultClassifyResponseCategory(BaseModel): + category: str + + +class ResultURLResult(BaseModel): + result_id: str + + type: Literal["url"] + """type = 'url'""" + + url: str + + +Result: TypeAlias = Union[ResultClassifyResponseCategory, ResultURLResult] + + +class ResponseConfidenceCategoryCriteriaConfidence(BaseModel): + """Confidence result for a single criterion.""" + + confidence: Literal["high", "low"] + + criterion: str + + +class ResponseConfidenceCategory(BaseModel): + """Confidence result for a category.""" + + category: str + + confidence: float + + criteria_confidence: List[ResponseConfidenceCategoryCriteriaConfidence] + + +class ResponseConfidence(BaseModel): + """Overall confidence breakdown for classification response.""" + + categories: List[ResponseConfidenceCategory] + + +class UsageUsageBreakdown(BaseModel): + """Raw classify quantities for accounts on the new pricing model. + + ``classify_pages`` is capped at 5, the same cap that the classify + credit computation uses. + """ + + classify_model: Literal["Classify", "Deep Classify"] + + classify_pages: Optional[int] = None + + +class Usage(BaseModel): + num_categories: int + + num_pages: int + + credits: Optional[float] = None + + usage_breakdown: Optional[UsageUsageBreakdown] = None + """Raw classify quantities for accounts on the new pricing model. + + `classify_pages` is capped at 5, the same cap that the classify credit + computation uses. + """ + + +class ClassifyResponse(BaseModel): + """Response from classify job - returned when polling /job/{job_id}""" + + job_id: str + + result: Result + + duration: Optional[float] = None + """The duration of the classify request in seconds.""" + + response_confidence: Optional[ResponseConfidence] = None + """Overall confidence breakdown for classification response.""" + + response_type: Optional[Literal["classify"]] = None + + usage: Optional[Usage] = None diff --git a/src/reducto/types/shared/direct_webhook_config.py b/src/reducto/types/shared/direct_webhook_config.py new file mode 100644 index 00000000..b0bff755 --- /dev/null +++ b/src/reducto/types/shared/direct_webhook_config.py @@ -0,0 +1,14 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional +from typing_extensions import Literal + +from ..._models import BaseModel + +__all__ = ["DirectWebhookConfig"] + + +class DirectWebhookConfig(BaseModel): + url: str + + mode: Optional[Literal["direct"]] = None diff --git a/src/reducto/types/edit_response.py b/src/reducto/types/shared/edit_response.py similarity index 67% rename from src/reducto/types/edit_response.py rename to src/reducto/types/shared/edit_response.py index 91c3217f..4709ac24 100644 --- a/src/reducto/types/edit_response.py +++ b/src/reducto/types/shared/edit_response.py @@ -1,10 +1,11 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. from typing import List, Optional +from typing_extensions import Literal -from .._models import BaseModel -from .edit_widget import EditWidget -from .parse_usage import ParseUsage +from ..._models import BaseModel +from ..edit_widget import EditWidget +from ..parse_usage import ParseUsage __all__ = ["EditResponse"] @@ -19,6 +20,11 @@ class EditResponse(BaseModel): List of widgets with their types, descriptions, and bounding boxes. """ + job_id: Optional[str] = None + """The unique identifier for the edit job.""" + + response_type: Optional[Literal["edit"]] = None + usage: Optional[ParseUsage] = None """ Usage information for the edit operation, including number of pages and credits diff --git a/src/reducto/types/shared/extract_response.py b/src/reducto/types/shared/extract_response.py new file mode 100644 index 00000000..5dcc8da8 --- /dev/null +++ b/src/reducto/types/shared/extract_response.py @@ -0,0 +1,65 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Dict, List, Union, Optional +from typing_extensions import Literal, TypeAlias + +from ..._models import BaseModel +from ..extract_usage import ExtractUsage + +__all__ = ["ExtractResponse", "Citations", "CitationsURLResult", "Result", "ResultURLResult"] + + +class CitationsURLResult(BaseModel): + result_id: str + + type: Literal["url"] + """type = 'url'""" + + url: str + + +Citations: TypeAlias = Union[List[object], CitationsURLResult, None] + + +class ResultURLResult(BaseModel): + result_id: str + + type: Literal["url"] + """type = 'url'""" + + url: str + + +Result: TypeAlias = Union[List[object], ResultURLResult] + + +class ExtractResponse(BaseModel): + citations: Optional[Citations] = None + """The citations corresponding to the extracted response. + + If force_url_result is True and citations are present, this is returned as a URL + result. + """ + + result: Result + """The extracted response in your provided schema. + + This is a list of dictionaries. If disable_chunking is True (default), then it + will be a list of length one. If force_url_result is True, this is returned as a + URL result. + """ + + usage: ExtractUsage + + job_id: Optional[str] = None + + response_confidence: Optional[Dict[str, object]] = None + """ + Optional deep extract confidence metadata containing document-level confidence + plus a mirrored leaf-level confidence tree. + """ + + response_type: Optional[Literal["extract"]] = None + + studio_link: Optional[str] = None + """The link to the studio pipeline for the document.""" diff --git a/src/reducto/types/shared/figure_agentic.py b/src/reducto/types/shared/figure_agentic.py new file mode 100644 index 00000000..1b60da8f --- /dev/null +++ b/src/reducto/types/shared/figure_agentic.py @@ -0,0 +1,30 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional +from typing_extensions import Literal + +from ..._models import BaseModel + +__all__ = ["FigureAgentic"] + + +class FigureAgentic(BaseModel): + scope: Literal["figure"] + + advanced_chart_agent: Optional[bool] = None + """ + If True, run advanced chart extraction on figures classified as charts: an + agentic extractor that returns full structured series data (chart_data) plus a + reconstruction image re-drawn from that data (extra.chart_reconstruction). + Higher latency. Defaults to False. + """ + + prompt: Optional[str] = None + """Custom prompt for figure agentic.""" + + return_overlays: Optional[bool] = None + """If True, return overlays for the figure. + + This is so you can use the overlays to double check the quality of the + extraction + """ diff --git a/src/reducto/types/shared/page_range.py b/src/reducto/types/shared/page_range.py new file mode 100644 index 00000000..9ecdd60a --- /dev/null +++ b/src/reducto/types/shared/page_range.py @@ -0,0 +1,15 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional + +from ..._models import BaseModel + +__all__ = ["PageRange"] + + +class PageRange(BaseModel): + end: Optional[int] = None + """The page number to stop processing at (1-indexed).""" + + start: Optional[int] = None + """The page number to start processing from (1-indexed).""" diff --git a/src/reducto/types/parse_response.py b/src/reducto/types/shared/parse_response.py similarity index 70% rename from src/reducto/types/parse_response.py rename to src/reducto/types/shared/parse_response.py index d9fa50da..6e4bda4e 100644 --- a/src/reducto/types/parse_response.py +++ b/src/reducto/types/shared/parse_response.py @@ -3,9 +3,9 @@ from typing import Dict, List, Union, Optional from typing_extensions import Literal, TypeAlias -from .._models import BaseModel -from .parse_usage import ParseUsage -from .bounding_box import BoundingBox +from ..._models import BaseModel +from ..parse_usage import ParseUsage +from ..bounding_box import BoundingBox __all__ = [ "ParseResponse", @@ -14,10 +14,12 @@ "ResultFullResultChunk", "ResultFullResultChunkBlock", "ResultFullResultChunkBlockGranularConfidence", + "ResultFullResultChunkBlockMergedTable", "ResultFullResultOcr", "ResultFullResultOcrLine", "ResultFullResultOcrWord", "ResultURLResult", + "DocumentProperties", ] @@ -32,6 +34,20 @@ class ResultFullResultChunkBlockGranularConfidence(BaseModel): parse_confidence: Optional[float] = None +class ResultFullResultChunkBlockMergedTable(BaseModel): + bbox: BoundingBox + """The original bounding box of a table before merge_tables merged it.""" + + content: str + """The original content of a table before merge_tables merged it.""" + + image_url: Optional[str] = None + """(Experimental) The URL of the image for this original table fragment. + + Only populated when settings.return_images includes 'table'. + """ + + class ResultFullResultChunkBlock(BaseModel): bbox: BoundingBox """The bounding box of the block extracted from the document.""" @@ -84,6 +100,9 @@ class ResultFullResultChunkBlock(BaseModel): image_url: Optional[str] = None """(Experimental) The URL of the image associated with the block.""" + merged_tables: Optional[List[ResultFullResultChunkBlockMergedTable]] = None + """Original table fragments that were combined into this table by merge_tables.""" + class ResultFullResultChunk(BaseModel): blocks: List[ResultFullResultChunkBlock] @@ -160,6 +179,43 @@ class ResultURLResult(BaseModel): Result: TypeAlias = Union[ResultFullResult, ResultURLResult] +class DocumentProperties(BaseModel): + """Embedded properties read from the customer's original document.""" + + author: Optional[str] = None + """The document author.""" + + created_at: Optional[str] = None + """The document creation time as a timezone-aware datetime. + + Dates without an offset are interpreted as UTC. + """ + + creator: Optional[str] = None + """The application or tool that authored the document.""" + + keywords: Optional[str] = None + """Keywords embedded in the document.""" + + last_modified_by: Optional[str] = None + """The user who last modified the document.""" + + modified_at: Optional[str] = None + """The document modification time as a timezone-aware datetime. + + Dates without an offset are interpreted as UTC. + """ + + producer: Optional[str] = None + """The application or library that produced the document.""" + + subject: Optional[str] = None + """The document subject.""" + + title: Optional[str] = None + """The document title.""" + + class ParseResponse(BaseModel): duration: float """The duration of the parse request in seconds.""" @@ -177,8 +233,13 @@ class ParseResponse(BaseModel): usage: ParseUsage + document_properties: Optional[DocumentProperties] = None + """Embedded properties read from the customer's original document.""" + pdf_url: Optional[str] = None """The storage URL of the converted PDF file.""" + response_type: Optional[Literal["parse"]] = None + studio_link: Optional[str] = None """The link to the studio pipeline for the document.""" diff --git a/src/reducto/types/pipeline_response.py b/src/reducto/types/shared/pipeline_response.py similarity index 62% rename from src/reducto/types/pipeline_response.py rename to src/reducto/types/shared/pipeline_response.py index cd7faafb..a08b50f5 100644 --- a/src/reducto/types/pipeline_response.py +++ b/src/reducto/types/shared/pipeline_response.py @@ -1,11 +1,11 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. from typing import List, Union, Optional -from typing_extensions import TypeAlias +from typing_extensions import Literal, TypeAlias -from .._models import BaseModel -from .v3_extract import V3Extract -from .parse_usage import ParseUsage +from ..._models import BaseModel +from ..v3_extract import V3Extract +from ..parse_usage import ParseUsage from .edit_response import EditResponse from .parse_response import ParseResponse from .split_response import SplitResponse @@ -15,27 +15,27 @@ "PipelineResponse", "Result", "ResultExtract", - "ResultExtractExtractVariant0", - "ResultExtractExtractVariant0Result", + "ResultExtractUnionMember0", + "ResultExtractUnionMember0Result", "ResultParse", ] -ResultExtractExtractVariant0Result: TypeAlias = Union[ExtractResponse, V3Extract] +ResultExtractUnionMember0Result: TypeAlias = Union[ExtractResponse, V3Extract] -class ResultExtractExtractVariant0(BaseModel): +class ResultExtractUnionMember0(BaseModel): """This is the response format for Extract -> Split Pipelines""" page_range: List[int] - result: ResultExtractExtractVariant0Result + result: ResultExtractUnionMember0Result split_name: str partition: Optional[str] = None -ResultExtract: TypeAlias = Union[List[ResultExtractExtractVariant0], ExtractResponse, V3Extract, None] +ResultExtract: TypeAlias = Union[List[ResultExtractUnionMember0], ExtractResponse, V3Extract, None] ResultParse: TypeAlias = Union[ParseResponse, List[ParseResponse], None] @@ -56,3 +56,5 @@ class PipelineResponse(BaseModel): result: Result usage: ParseUsage + + response_type: Optional[Literal["pipeline"]] = None diff --git a/src/reducto/types/shared/split_large_tables.py b/src/reducto/types/shared/split_large_tables.py new file mode 100644 index 00000000..33409b77 --- /dev/null +++ b/src/reducto/types/shared/split_large_tables.py @@ -0,0 +1,38 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Union, Optional +from typing_extensions import TypeAlias + +from ..._models import BaseModel + +__all__ = ["SplitLargeTables", "Size", "SizeSplitLargeTableSizes"] + + +class SizeSplitLargeTableSizes(BaseModel): + column: Optional[int] = None + """The number of columns to include in each chunk when splitting large tables. + + Does not chunk columns if set to None. + """ + + row: Optional[int] = None + """The number of rows to include in each chunk when splitting large tables. + + Does not chunk rows if set to None. + """ + + +Size: TypeAlias = Union[int, SizeSplitLargeTableSizes] + + +class SplitLargeTables(BaseModel): + enabled: Optional[bool] = None + """If True, split large tables into smaller tables. Defaults to True.""" + + size: Optional[Size] = None + """The size of the tables to split into. + + Defaults to 50. Use 'row' and 'column' to independently specify the number of + rows and columns to include when splitting. If you only want to split by rows or + columns, set the other value to None. + """ diff --git a/src/reducto/types/split_response.py b/src/reducto/types/shared/split_response.py similarity index 69% rename from src/reducto/types/split_response.py rename to src/reducto/types/shared/split_response.py index b3544af4..63eb591b 100644 --- a/src/reducto/types/split_response.py +++ b/src/reducto/types/shared/split_response.py @@ -3,9 +3,9 @@ from typing import Dict, List, Union, Optional from typing_extensions import Literal, TypeAlias -from .._models import BaseModel -from .parse_usage import ParseUsage -from .deep_split_page_evidence import DeepSplitPageEvidence +from ..._models import BaseModel +from ..parse_usage import ParseUsage +from ..deep_split_page_evidence import DeepSplitPageEvidence __all__ = [ "SplitResponse", @@ -16,6 +16,7 @@ "ResultDeepSplitResult", "ResultDeepSplitResultSplit", "ResultDeepSplitResultSplitPartition", + "ResultURLResult", ] @@ -61,11 +62,31 @@ class ResultDeepSplitResult(BaseModel): splits: List[ResultDeepSplitResultSplit] -Result: TypeAlias = Union[ResultSplitResult, ResultDeepSplitResult] +class ResultURLResult(BaseModel): + result_id: str + + type: Literal["url"] + """type = 'url'""" + + url: str + + +Result: TypeAlias = Union[ResultSplitResult, ResultDeepSplitResult, ResultURLResult] class SplitResponse(BaseModel): result: Result - """The split result.""" + """The split result. + + If force_url_result is True, this is returned as a URL result. + """ usage: ParseUsage + + duration: Optional[float] = None + """The duration of the split request in seconds.""" + + job_id: Optional[str] = None + """The unique identifier for the split job.""" + + response_type: Optional[Literal["split"]] = None diff --git a/src/reducto/types/shared/svix_webhook_config.py b/src/reducto/types/shared/svix_webhook_config.py new file mode 100644 index 00000000..4cbb5627 --- /dev/null +++ b/src/reducto/types/shared/svix_webhook_config.py @@ -0,0 +1,18 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import List, Optional +from typing_extensions import Literal + +from ..._models import BaseModel + +__all__ = ["SvixWebhookConfig"] + + +class SvixWebhookConfig(BaseModel): + channels: Optional[List[str]] = None + """ + A list of Svix channels the message will be delivered down, omit to send to all + channels. + """ + + mode: Optional[Literal["svix"]] = None diff --git a/src/reducto/types/shared/table_agentic.py b/src/reducto/types/shared/table_agentic.py new file mode 100644 index 00000000..b43c93fe --- /dev/null +++ b/src/reducto/types/shared/table_agentic.py @@ -0,0 +1,21 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional +from typing_extensions import Literal + +from ..._models import BaseModel + +__all__ = ["TableAgentic"] + + +class TableAgentic(BaseModel): + scope: Literal["table"] + + mode: Optional[Literal["default", "auto", "max"]] = None + """ + Mode for table agentic: 'default' selectively applies enrichment only to tables + likely to benefit, and 'max' runs enrichment on all tables. + """ + + prompt: Optional[str] = None + """Custom prompt for table agentic.""" diff --git a/src/reducto/types/shared/text_agentic.py b/src/reducto/types/shared/text_agentic.py new file mode 100644 index 00000000..e2074786 --- /dev/null +++ b/src/reducto/types/shared/text_agentic.py @@ -0,0 +1,18 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import Optional +from typing_extensions import Literal + +from ..._models import BaseModel + +__all__ = ["TextAgentic"] + + +class TextAgentic(BaseModel): + scope: Literal["text"] + + prompt: Optional[str] = None + """Custom instructions for agentic text. + + Note: This only applies to form regions (key-value). + """ diff --git a/src/reducto/types/shared/webhook_config_new.py b/src/reducto/types/shared/webhook_config_new.py new file mode 100644 index 00000000..e28c8b6d --- /dev/null +++ b/src/reducto/types/shared/webhook_config_new.py @@ -0,0 +1,28 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from typing import List, Optional +from typing_extensions import Literal + +from ..._models import BaseModel + +__all__ = ["WebhookConfigNew"] + + +class WebhookConfigNew(BaseModel): + channels: Optional[List[str]] = None + """ + A list of Svix channels the message will be delivered down, omit to send to all + channels. + """ + + metadata: Optional[object] = None + """JSON metadata included in webhook request body""" + + mode: Optional[Literal["disabled", "svix", "direct"]] = None + """The mode to use for webhook delivery. + + Defaults to 'disabled'. We recommend using 'svix' for production environments. + """ + + url: Optional[str] = None + """The URL to send the webhook to (if using direct webhoook).""" diff --git a/src/reducto/types/shared_params/__init__.py b/src/reducto/types/shared_params/__init__.py index e9bf9399..4a4299d3 100644 --- a/src/reducto/types/shared_params/__init__.py +++ b/src/reducto/types/shared_params/__init__.py @@ -1,3 +1,12 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. from .upload import Upload as Upload +from .chunking import Chunking as Chunking +from .page_range import PageRange as PageRange +from .text_agentic import TextAgentic as TextAgentic +from .table_agentic import TableAgentic as TableAgentic +from .figure_agentic import FigureAgentic as FigureAgentic +from .split_large_tables import SplitLargeTables as SplitLargeTables +from .webhook_config_new import WebhookConfigNew as WebhookConfigNew +from .svix_webhook_config import SvixWebhookConfig as SvixWebhookConfig +from .direct_webhook_config import DirectWebhookConfig as DirectWebhookConfig diff --git a/src/reducto/types/shared_params/chunking.py b/src/reducto/types/shared_params/chunking.py new file mode 100644 index 00000000..2d79eda7 --- /dev/null +++ b/src/reducto/types/shared_params/chunking.py @@ -0,0 +1,29 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Optional +from typing_extensions import Literal, TypedDict + +__all__ = ["Chunking"] + + +class Chunking(TypedDict, total=False): + chunk_mode: Literal["variable", "section", "page", "disabled", "block", "page_sections"] + """Choose how to partition chunks. + + Variable mode chunks by character length and visual context. Section mode chunks + by section headers. Page mode chunks according to pages. Page sections mode + chunks first by page, then by sections within each page. Disabled returns one + single chunk. + """ + + chunk_overlap: int + """Number of characters of overlap to include from adjacent chunks. Defaults to 0.""" + + chunk_size: Optional[int] + """ + The approximate size of chunks (in characters) that the document will be split + into. Defaults to null, in which case the chunk size is variable between 250 - + 1500 characters. + """ diff --git a/src/reducto/types/shared_params/direct_webhook_config.py b/src/reducto/types/shared_params/direct_webhook_config.py new file mode 100644 index 00000000..841219f1 --- /dev/null +++ b/src/reducto/types/shared_params/direct_webhook_config.py @@ -0,0 +1,13 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Literal, Required, TypedDict + +__all__ = ["DirectWebhookConfig"] + + +class DirectWebhookConfig(TypedDict, total=False): + url: Required[str] + + mode: Literal["direct"] diff --git a/src/reducto/types/shared_params/figure_agentic.py b/src/reducto/types/shared_params/figure_agentic.py new file mode 100644 index 00000000..5fe9d227 --- /dev/null +++ b/src/reducto/types/shared_params/figure_agentic.py @@ -0,0 +1,30 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Optional +from typing_extensions import Literal, Required, TypedDict + +__all__ = ["FigureAgentic"] + + +class FigureAgentic(TypedDict, total=False): + scope: Required[Literal["figure"]] + + advanced_chart_agent: bool + """ + If True, run advanced chart extraction on figures classified as charts: an + agentic extractor that returns full structured series data (chart_data) plus a + reconstruction image re-drawn from that data (extra.chart_reconstruction). + Higher latency. Defaults to False. + """ + + prompt: Optional[str] + """Custom prompt for figure agentic.""" + + return_overlays: bool + """If True, return overlays for the figure. + + This is so you can use the overlays to double check the quality of the + extraction + """ diff --git a/src/reducto/types/page_range_param.py b/src/reducto/types/shared_params/page_range.py similarity index 83% rename from src/reducto/types/page_range_param.py rename to src/reducto/types/shared_params/page_range.py index d2b9e2a5..780830b7 100644 --- a/src/reducto/types/page_range_param.py +++ b/src/reducto/types/shared_params/page_range.py @@ -5,10 +5,10 @@ from typing import Optional from typing_extensions import TypedDict -__all__ = ["PageRangeParam"] +__all__ = ["PageRange"] -class PageRangeParam(TypedDict, total=False): +class PageRange(TypedDict, total=False): end: Optional[int] """The page number to stop processing at (1-indexed).""" diff --git a/src/reducto/types/shared_params/split_large_tables.py b/src/reducto/types/shared_params/split_large_tables.py new file mode 100644 index 00000000..82c1c45b --- /dev/null +++ b/src/reducto/types/shared_params/split_large_tables.py @@ -0,0 +1,38 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Union, Optional +from typing_extensions import TypeAlias, TypedDict + +__all__ = ["SplitLargeTables", "Size", "SizeSplitLargeTableSizes"] + + +class SizeSplitLargeTableSizes(TypedDict, total=False): + column: Optional[int] + """The number of columns to include in each chunk when splitting large tables. + + Does not chunk columns if set to None. + """ + + row: Optional[int] + """The number of rows to include in each chunk when splitting large tables. + + Does not chunk rows if set to None. + """ + + +Size: TypeAlias = Union[int, SizeSplitLargeTableSizes] + + +class SplitLargeTables(TypedDict, total=False): + enabled: bool + """If True, split large tables into smaller tables. Defaults to True.""" + + size: Size + """The size of the tables to split into. + + Defaults to 50. Use 'row' and 'column' to independently specify the number of + rows and columns to include when splitting. If you only want to split by rows or + columns, set the other value to None. + """ diff --git a/src/reducto/types/shared_params/svix_webhook_config.py b/src/reducto/types/shared_params/svix_webhook_config.py new file mode 100644 index 00000000..cf571641 --- /dev/null +++ b/src/reducto/types/shared_params/svix_webhook_config.py @@ -0,0 +1,19 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Literal, TypedDict + +from ..._types import SequenceNotStr + +__all__ = ["SvixWebhookConfig"] + + +class SvixWebhookConfig(TypedDict, total=False): + channels: SequenceNotStr[str] + """ + A list of Svix channels the message will be delivered down, omit to send to all + channels. + """ + + mode: Literal["svix"] diff --git a/src/reducto/types/shared_params/table_agentic.py b/src/reducto/types/shared_params/table_agentic.py new file mode 100644 index 00000000..2f918bbb --- /dev/null +++ b/src/reducto/types/shared_params/table_agentic.py @@ -0,0 +1,21 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Optional +from typing_extensions import Literal, Required, TypedDict + +__all__ = ["TableAgentic"] + + +class TableAgentic(TypedDict, total=False): + scope: Required[Literal["table"]] + + mode: Literal["default", "auto", "max"] + """ + Mode for table agentic: 'default' selectively applies enrichment only to tables + likely to benefit, and 'max' runs enrichment on all tables. + """ + + prompt: Optional[str] + """Custom prompt for table agentic.""" diff --git a/src/reducto/types/shared_params/text_agentic.py b/src/reducto/types/shared_params/text_agentic.py new file mode 100644 index 00000000..be3ae8b2 --- /dev/null +++ b/src/reducto/types/shared_params/text_agentic.py @@ -0,0 +1,18 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing import Optional +from typing_extensions import Literal, Required, TypedDict + +__all__ = ["TextAgentic"] + + +class TextAgentic(TypedDict, total=False): + scope: Required[Literal["text"]] + + prompt: Optional[str] + """Custom instructions for agentic text. + + Note: This only applies to form regions (key-value). + """ diff --git a/src/reducto/types/shared_params/webhook_config_new.py b/src/reducto/types/shared_params/webhook_config_new.py new file mode 100644 index 00000000..0ebd8e3c --- /dev/null +++ b/src/reducto/types/shared_params/webhook_config_new.py @@ -0,0 +1,29 @@ +# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. + +from __future__ import annotations + +from typing_extensions import Literal, TypedDict + +from ..._types import SequenceNotStr + +__all__ = ["WebhookConfigNew"] + + +class WebhookConfigNew(TypedDict, total=False): + channels: SequenceNotStr[str] + """ + A list of Svix channels the message will be delivered down, omit to send to all + channels. + """ + + metadata: object + """JSON metadata included in webhook request body""" + + mode: Literal["disabled", "svix", "direct"] + """The mode to use for webhook delivery. + + Defaults to 'disabled'. We recommend using 'svix' for production environments. + """ + + url: str + """The URL to send the webhook to (if using direct webhoook).""" diff --git a/src/reducto/types/split_run_job_params.py b/src/reducto/types/split_run_job_params.py index f21ae1f8..4f03af2b 100644 --- a/src/reducto/types/split_run_job_params.py +++ b/src/reducto/types/split_run_job_params.py @@ -3,7 +3,7 @@ from __future__ import annotations from typing import Union, Iterable -from typing_extensions import Required, Annotated, TypeAlias, TypedDict +from typing_extensions import Literal, Required, Annotated, TypeAlias, TypedDict from .._types import SequenceNotStr from .._utils import PropertyInfo @@ -11,9 +11,8 @@ from .shared_params.upload import Upload from .split_category_param import SplitCategoryParam from .async_config_v3_param import AsyncConfigV3Param -from .split_table_options_param import SplitTableOptionsParam -__all__ = ["SplitRunJobParams", "Input"] +__all__ = ["SplitRunJobParams", "Input", "Settings"] class SplitRunJobParams(TypedDict, total=False): @@ -42,7 +41,7 @@ class SplitRunJobParams(TypedDict, total=False): be ignored. """ - settings: SplitTableOptionsParam + settings: Settings """The settings for split processing.""" split_rules: str @@ -50,3 +49,38 @@ class SplitRunJobParams(TypedDict, total=False): Input: TypeAlias = Union[str, SequenceNotStr[str], Upload] + + +class Settings(TypedDict, total=False): + """The settings for split processing.""" + + allow_page_overlap: bool + """If True, a page can belong to multiple categories/partitions. + + If False, each page must belong to exactly one category. Defaults to True. + """ + + auto_partition: bool + """ + If True (default), deep split may split a category into partitions even when + that category has no configured partition_key. If False, categories without a + partition_key are never partitioned, so partitioning happens only where you + explicitly configured a partition_key. + """ + + deep_split: bool + """If True, uses the deep split agent for higher-quality document splitting. + + Off by default. + """ + + force_url_result: bool + """Force the endpoint result to be returned in URL form.""" + + table_cutoff: Literal["truncate", "preserve"] + """ + If tables should be truncated to the first few rows or if all content should be + preserved. truncate improves latency, preserve is recommended for cases where + partition_key is being used and the partition_key may be included within the + table. Defaults to truncate + """ diff --git a/src/reducto/types/split_run_params.py b/src/reducto/types/split_run_params.py index ae6fbeee..b931f244 100644 --- a/src/reducto/types/split_run_params.py +++ b/src/reducto/types/split_run_params.py @@ -3,15 +3,14 @@ from __future__ import annotations from typing import Union, Iterable -from typing_extensions import Required, TypeAlias, TypedDict +from typing_extensions import Literal, Required, TypeAlias, TypedDict from .._types import SequenceNotStr from .parse_options_param import ParseOptionsParam from .shared_params.upload import Upload from .split_category_param import SplitCategoryParam -from .split_table_options_param import SplitTableOptionsParam -__all__ = ["SplitRunParams", "Input"] +__all__ = ["SplitRunParams", "Input", "Settings"] class SplitRunParams(TypedDict, total=False): @@ -37,7 +36,7 @@ class SplitRunParams(TypedDict, total=False): be ignored. """ - settings: SplitTableOptionsParam + settings: Settings """The settings for split processing.""" split_rules: str @@ -45,3 +44,38 @@ class SplitRunParams(TypedDict, total=False): Input: TypeAlias = Union[str, SequenceNotStr[str], Upload] + + +class Settings(TypedDict, total=False): + """The settings for split processing.""" + + allow_page_overlap: bool + """If True, a page can belong to multiple categories/partitions. + + If False, each page must belong to exactly one category. Defaults to True. + """ + + auto_partition: bool + """ + If True (default), deep split may split a category into partitions even when + that category has no configured partition_key. If False, categories without a + partition_key are never partitioned, so partitioning happens only where you + explicitly configured a partition_key. + """ + + deep_split: bool + """If True, uses the deep split agent for higher-quality document splitting. + + Off by default. + """ + + force_url_result: bool + """Force the endpoint result to be returned in URL form.""" + + table_cutoff: Literal["truncate", "preserve"] + """ + If tables should be truncated to the first few rows or if all content should be + preserved. truncate improves latency, preserve is recommended for cases where + partition_key is being used and the partition_key may be included within the + table. Defaults to truncate + """ diff --git a/src/reducto/types/split_table_options_param.py b/src/reducto/types/split_table_options_param.py deleted file mode 100644 index 740e863c..00000000 --- a/src/reducto/types/split_table_options_param.py +++ /dev/null @@ -1,17 +0,0 @@ -# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. - -from __future__ import annotations - -from typing_extensions import Literal, TypedDict - -__all__ = ["SplitTableOptionsParam"] - - -class SplitTableOptionsParam(TypedDict, total=False): - table_cutoff: Literal["truncate", "preserve"] - """ - If tables should be truncated to the first few rows or if all content should be - preserved. truncate improves latency, preserve is recommended for cases where - partition_key is being used and the partition_key may be included within the - table. Defaults to truncate - """ diff --git a/src/reducto/types/spreadsheet_param.py b/src/reducto/types/spreadsheet_param.py index 4a3e1a12..6eb5765a 100644 --- a/src/reducto/types/spreadsheet_param.py +++ b/src/reducto/types/spreadsheet_param.py @@ -2,40 +2,12 @@ from __future__ import annotations -from typing import List, Union, Optional -from typing_extensions import Literal, TypeAlias, TypedDict +from typing import List, Optional +from typing_extensions import Literal, TypedDict -__all__ = ["SpreadsheetParam", "SplitLargeTables", "SplitLargeTablesSize", "SplitLargeTablesSizeSplitLargeTableSizes"] +from .shared_params.split_large_tables import SplitLargeTables - -class SplitLargeTablesSizeSplitLargeTableSizes(TypedDict, total=False): - column: Optional[int] - """The number of columns to include in each chunk when splitting large tables. - - Does not chunk columns if set to None. - """ - - row: Optional[int] - """The number of rows to include in each chunk when splitting large tables. - - Does not chunk rows if set to None. - """ - - -SplitLargeTablesSize: TypeAlias = Union[int, SplitLargeTablesSizeSplitLargeTableSizes] - - -class SplitLargeTables(TypedDict, total=False): - enabled: bool - """If True, split large tables into smaller tables. Defaults to True.""" - - size: SplitLargeTablesSize - """The size of the tables to split into. - - Defaults to 50. Use 'row' and 'column' to independently specify the number of - rows and columns to include when splitting. If you only want to split by rows or - columns, set the other value to None. - """ +__all__ = ["SpreadsheetParam"] class SpreadsheetParam(TypedDict, total=False): @@ -52,4 +24,11 @@ class SpreadsheetParam(TypedDict, total=False): include: List[Literal["cell_colors", "formula", "dropdowns"]] """Whether to include cell color, formula, and dropdown information in the output.""" + max_cell_count: Optional[int] + """Maximum total non-empty cells allowed across all sheets. + + If exceeded, the request is rejected with a 422 error. Set to null to disable + the limit. Defaults to null. + """ + split_large_tables: SplitLargeTables diff --git a/src/reducto/types/v3_extract.py b/src/reducto/types/v3_extract.py index e2b61eb0..31ec9722 100644 --- a/src/reducto/types/v3_extract.py +++ b/src/reducto/types/v3_extract.py @@ -1,6 +1,7 @@ # File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details. from typing import List, Union, Optional +from typing_extensions import Literal from .._models import BaseModel from .extract_usage import ExtractUsage @@ -18,7 +19,15 @@ class V3Extract(BaseModel): usage: ExtractUsage + confidence: Optional[Literal["high", "low"]] = None + """Optional document-level deep extract confidence label.""" + + confidence_reason: Optional[str] = None + """Optional explanation for the document-level confidence label.""" + job_id: Optional[str] = None + response_type: Optional[Literal["v3_extract"]] = None + studio_link: Optional[str] = None """The link to the studio pipeline for the document.""" diff --git a/tests/api_resources/test_classify.py b/tests/api_resources/test_classify.py index 04bdff54..d8aa3a43 100644 --- a/tests/api_resources/test_classify.py +++ b/tests/api_resources/test_classify.py @@ -9,7 +9,7 @@ from reducto import Reducto, AsyncReducto from tests.utils import assert_matches_type -from reducto.types import ClassifyResponse +from reducto.types.shared import ClassifyResponse base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010") @@ -37,11 +37,12 @@ def test_method_run_with_all_params(self, client: Reducto) -> None: } ], document_metadata="document_metadata", + force_url_result=True, page_range={ "end": 0, "start": 0, }, - persist_results=True, + priority=True, ) assert_matches_type(ClassifyResponse, classify, path=["response"]) @@ -97,11 +98,12 @@ async def test_method_run_with_all_params(self, async_client: AsyncReducto) -> N } ], document_metadata="document_metadata", + force_url_result=True, page_range={ "end": 0, "start": 0, }, - persist_results=True, + priority=True, ) assert_matches_type(ClassifyResponse, classify, path=["response"]) diff --git a/tests/api_resources/test_client.py b/tests/api_resources/test_client.py index 66d4adb5..4d11b754 100644 --- a/tests/api_resources/test_client.py +++ b/tests/api_resources/test_client.py @@ -56,7 +56,7 @@ def test_method_upload(self, client: Reducto) -> None: def test_method_upload_with_all_params(self, client: Reducto) -> None: client_ = client.upload( extension="extension", - file="file", + file=b"Example data", ) assert_matches_type(Upload, client_, path=["response"]) @@ -127,7 +127,7 @@ async def test_method_upload(self, async_client: AsyncReducto) -> None: async def test_method_upload_with_all_params(self, async_client: AsyncReducto) -> None: client = await async_client.upload( extension="extension", - file="file", + file=b"Example data", ) assert_matches_type(Upload, client, path=["response"]) diff --git a/tests/api_resources/test_edit.py b/tests/api_resources/test_edit.py index 426d8c2c..c47b3356 100644 --- a/tests/api_resources/test_edit.py +++ b/tests/api_resources/test_edit.py @@ -9,7 +9,7 @@ from reducto import Reducto, AsyncReducto from tests.utils import assert_matches_type -from reducto.types import EditResponse, EditRunJobResponse +from reducto.types.shared import EditResponse, AsyncEditResponse base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010") @@ -95,7 +95,7 @@ def test_method_run_job(self, client: Reducto) -> None: document_url="string", edit_instructions="edit_instructions", ) - assert_matches_type(EditRunJobResponse, edit, path=["response"]) + assert_matches_type(AsyncEditResponse, edit, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize @@ -135,7 +135,7 @@ def test_method_run_job_with_all_params(self, client: Reducto) -> None: "url": "url", }, ) - assert_matches_type(EditRunJobResponse, edit, path=["response"]) + assert_matches_type(AsyncEditResponse, edit, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize @@ -148,7 +148,7 @@ def test_raw_response_run_job(self, client: Reducto) -> None: assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" edit = response.parse() - assert_matches_type(EditRunJobResponse, edit, path=["response"]) + assert_matches_type(AsyncEditResponse, edit, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize @@ -161,7 +161,7 @@ def test_streaming_response_run_job(self, client: Reducto) -> None: assert response.http_request.headers.get("X-Stainless-Lang") == "python" edit = response.parse() - assert_matches_type(EditRunJobResponse, edit, path=["response"]) + assert_matches_type(AsyncEditResponse, edit, path=["response"]) assert cast(Any, response.is_closed) is True @@ -249,7 +249,7 @@ async def test_method_run_job(self, async_client: AsyncReducto) -> None: document_url="string", edit_instructions="edit_instructions", ) - assert_matches_type(EditRunJobResponse, edit, path=["response"]) + assert_matches_type(AsyncEditResponse, edit, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize @@ -289,7 +289,7 @@ async def test_method_run_job_with_all_params(self, async_client: AsyncReducto) "url": "url", }, ) - assert_matches_type(EditRunJobResponse, edit, path=["response"]) + assert_matches_type(AsyncEditResponse, edit, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize @@ -302,7 +302,7 @@ async def test_raw_response_run_job(self, async_client: AsyncReducto) -> None: assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" edit = await response.parse() - assert_matches_type(EditRunJobResponse, edit, path=["response"]) + assert_matches_type(AsyncEditResponse, edit, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize @@ -315,6 +315,6 @@ async def test_streaming_response_run_job(self, async_client: AsyncReducto) -> N assert response.http_request.headers.get("X-Stainless-Lang") == "python" edit = await response.parse() - assert_matches_type(EditRunJobResponse, edit, path=["response"]) + assert_matches_type(AsyncEditResponse, edit, path=["response"]) assert cast(Any, response.is_closed) is True diff --git a/tests/api_resources/test_extract.py b/tests/api_resources/test_extract.py index 8a3a4202..a25014c2 100644 --- a/tests/api_resources/test_extract.py +++ b/tests/api_resources/test_extract.py @@ -11,8 +11,8 @@ from tests.utils import assert_matches_type from reducto.types import ( ExtractRunResponse, - AsyncExtractResponse, ) +from reducto.types.shared import AsyncExtractResponse base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010") @@ -39,9 +39,11 @@ def test_method_run_with_all_params_overload_1(self, client: Reducto) -> None: }, parsing={ "enhance": { + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -66,9 +68,13 @@ def test_method_run_with_all_params_overload_1(self, client: Reducto) -> None: "settings": { "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, + "model": "r-1", "ocr_system": "standard", "page_range": { "end": 0, @@ -77,12 +83,17 @@ def test_method_run_with_all_params_overload_1(self, client: Reducto) -> None: "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "x", + "max_share": 1, + }, "timeout": 0, }, "spreadsheet": { "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 1, "split_large_tables": { "enabled": True, "size": 0, @@ -94,10 +105,16 @@ def test_method_run_with_all_params_overload_1(self, client: Reducto) -> None: "citations": { "enabled": True, "numerical_confidence": True, + "parent_block": "full", }, "deep_extract": True, + "force_url_result": True, "include_images": True, "optimize_for_latency": True, + "page_range": { + "end": 0, + "start": 0, + }, }, ) assert_matches_type(ExtractRunResponse, extract, path=["response"]) @@ -155,9 +172,11 @@ def test_method_run_with_all_params_overload_2(self, client: Reducto) -> None: }, parsing={ "enhance": { + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -182,9 +201,13 @@ def test_method_run_with_all_params_overload_2(self, client: Reducto) -> None: "settings": { "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, + "model": "r-1", "ocr_system": "standard", "page_range": { "end": 0, @@ -193,12 +216,17 @@ def test_method_run_with_all_params_overload_2(self, client: Reducto) -> None: "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "x", + "max_share": 1, + }, "timeout": 0, }, "spreadsheet": { "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 1, "split_large_tables": { "enabled": True, "size": 0, @@ -210,10 +238,16 @@ def test_method_run_with_all_params_overload_2(self, client: Reducto) -> None: "citations": { "enabled": True, "numerical_confidence": True, + "parent_block": "full", }, "deep_extract": True, + "force_url_result": True, "include_images": True, "optimize_for_latency": True, + "page_range": { + "end": 0, + "start": 0, + }, }, ) assert_matches_type(ExtractRunResponse, extract, path=["response"]) @@ -271,9 +305,11 @@ def test_method_run_job_with_all_params(self, client: Reducto) -> None: }, parsing={ "enhance": { + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -298,9 +334,13 @@ def test_method_run_job_with_all_params(self, client: Reducto) -> None: "settings": { "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, + "model": "r-1", "ocr_system": "standard", "page_range": { "end": 0, @@ -309,12 +349,17 @@ def test_method_run_job_with_all_params(self, client: Reducto) -> None: "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "x", + "max_share": 1, + }, "timeout": 0, }, "spreadsheet": { "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 1, "split_large_tables": { "enabled": True, "size": 0, @@ -326,10 +371,16 @@ def test_method_run_job_with_all_params(self, client: Reducto) -> None: "citations": { "enabled": True, "numerical_confidence": True, + "parent_block": "full", }, "deep_extract": True, + "force_url_result": True, "include_images": True, "optimize_for_latency": True, + "page_range": { + "end": 0, + "start": 0, + }, }, ) assert_matches_type(AsyncExtractResponse, extract, path=["response"]) @@ -385,9 +436,11 @@ async def test_method_run_with_all_params_overload_1(self, async_client: AsyncRe }, parsing={ "enhance": { + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -412,9 +465,13 @@ async def test_method_run_with_all_params_overload_1(self, async_client: AsyncRe "settings": { "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, + "model": "r-1", "ocr_system": "standard", "page_range": { "end": 0, @@ -423,12 +480,17 @@ async def test_method_run_with_all_params_overload_1(self, async_client: AsyncRe "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "x", + "max_share": 1, + }, "timeout": 0, }, "spreadsheet": { "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 1, "split_large_tables": { "enabled": True, "size": 0, @@ -440,10 +502,16 @@ async def test_method_run_with_all_params_overload_1(self, async_client: AsyncRe "citations": { "enabled": True, "numerical_confidence": True, + "parent_block": "full", }, "deep_extract": True, + "force_url_result": True, "include_images": True, "optimize_for_latency": True, + "page_range": { + "end": 0, + "start": 0, + }, }, ) assert_matches_type(ExtractRunResponse, extract, path=["response"]) @@ -501,9 +569,11 @@ async def test_method_run_with_all_params_overload_2(self, async_client: AsyncRe }, parsing={ "enhance": { + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -528,9 +598,13 @@ async def test_method_run_with_all_params_overload_2(self, async_client: AsyncRe "settings": { "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, + "model": "r-1", "ocr_system": "standard", "page_range": { "end": 0, @@ -539,12 +613,17 @@ async def test_method_run_with_all_params_overload_2(self, async_client: AsyncRe "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "x", + "max_share": 1, + }, "timeout": 0, }, "spreadsheet": { "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 1, "split_large_tables": { "enabled": True, "size": 0, @@ -556,10 +635,16 @@ async def test_method_run_with_all_params_overload_2(self, async_client: AsyncRe "citations": { "enabled": True, "numerical_confidence": True, + "parent_block": "full", }, "deep_extract": True, + "force_url_result": True, "include_images": True, "optimize_for_latency": True, + "page_range": { + "end": 0, + "start": 0, + }, }, ) assert_matches_type(ExtractRunResponse, extract, path=["response"]) @@ -617,9 +702,11 @@ async def test_method_run_job_with_all_params(self, async_client: AsyncReducto) }, parsing={ "enhance": { + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -644,9 +731,13 @@ async def test_method_run_job_with_all_params(self, async_client: AsyncReducto) "settings": { "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, + "model": "r-1", "ocr_system": "standard", "page_range": { "end": 0, @@ -655,12 +746,17 @@ async def test_method_run_job_with_all_params(self, async_client: AsyncReducto) "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "x", + "max_share": 1, + }, "timeout": 0, }, "spreadsheet": { "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 1, "split_large_tables": { "enabled": True, "size": 0, @@ -672,10 +768,16 @@ async def test_method_run_job_with_all_params(self, async_client: AsyncReducto) "citations": { "enabled": True, "numerical_confidence": True, + "parent_block": "full", }, "deep_extract": True, + "force_url_result": True, "include_images": True, "optimize_for_latency": True, + "page_range": { + "end": 0, + "start": 0, + }, }, ) assert_matches_type(AsyncExtractResponse, extract, path=["response"]) diff --git a/tests/api_resources/test_parse.py b/tests/api_resources/test_parse.py index 4590de2b..b350deed 100644 --- a/tests/api_resources/test_parse.py +++ b/tests/api_resources/test_parse.py @@ -11,8 +11,8 @@ from tests.utils import assert_matches_type from reducto.types import ( ParseRunResponse, - AsyncParseResponse, ) +from reducto.types.shared import AsyncParseResponse base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010") @@ -34,9 +34,11 @@ def test_method_run_with_all_params_overload_1(self, client: Reducto) -> None: parse = client.parse.run( input="string", enhance={ + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -61,9 +63,13 @@ def test_method_run_with_all_params_overload_1(self, client: Reducto) -> None: settings={ "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, + "model": "r-1", "ocr_system": "standard", "page_range": { "end": 0, @@ -72,12 +78,17 @@ def test_method_run_with_all_params_overload_1(self, client: Reducto) -> None: "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "x", + "max_share": 1, + }, "timeout": 0, }, spreadsheet={ "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 1, "split_large_tables": { "enabled": True, "size": 0, @@ -134,9 +145,11 @@ def test_method_run_with_all_params_overload_2(self, client: Reducto) -> None: }, }, enhance={ + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -162,9 +175,13 @@ def test_method_run_with_all_params_overload_2(self, client: Reducto) -> None: settings={ "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, + "model": "r-1", "ocr_system": "standard", "page_range": { "end": 0, @@ -173,12 +190,17 @@ def test_method_run_with_all_params_overload_2(self, client: Reducto) -> None: "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "x", + "max_share": 1, + }, "timeout": 0, }, spreadsheet={ "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 1, "split_large_tables": { "enabled": True, "size": 0, @@ -235,9 +257,11 @@ def test_method_run_job_with_all_params(self, client: Reducto) -> None: }, }, enhance={ + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -263,9 +287,13 @@ def test_method_run_job_with_all_params(self, client: Reducto) -> None: settings={ "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, + "model": "r-1", "ocr_system": "standard", "page_range": { "end": 0, @@ -274,12 +302,17 @@ def test_method_run_job_with_all_params(self, client: Reducto) -> None: "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "x", + "max_share": 1, + }, "timeout": 0, }, spreadsheet={ "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 1, "split_large_tables": { "enabled": True, "size": 0, @@ -334,9 +367,11 @@ async def test_method_run_with_all_params_overload_1(self, async_client: AsyncRe parse = await async_client.parse.run( input="string", enhance={ + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -361,9 +396,13 @@ async def test_method_run_with_all_params_overload_1(self, async_client: AsyncRe settings={ "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, + "model": "r-1", "ocr_system": "standard", "page_range": { "end": 0, @@ -372,12 +411,17 @@ async def test_method_run_with_all_params_overload_1(self, async_client: AsyncRe "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "x", + "max_share": 1, + }, "timeout": 0, }, spreadsheet={ "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 1, "split_large_tables": { "enabled": True, "size": 0, @@ -434,9 +478,11 @@ async def test_method_run_with_all_params_overload_2(self, async_client: AsyncRe }, }, enhance={ + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -462,9 +508,13 @@ async def test_method_run_with_all_params_overload_2(self, async_client: AsyncRe settings={ "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, + "model": "r-1", "ocr_system": "standard", "page_range": { "end": 0, @@ -473,12 +523,17 @@ async def test_method_run_with_all_params_overload_2(self, async_client: AsyncRe "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "x", + "max_share": 1, + }, "timeout": 0, }, spreadsheet={ "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 1, "split_large_tables": { "enabled": True, "size": 0, @@ -535,9 +590,11 @@ async def test_method_run_job_with_all_params(self, async_client: AsyncReducto) }, }, enhance={ + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -563,9 +620,13 @@ async def test_method_run_job_with_all_params(self, async_client: AsyncReducto) settings={ "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, + "model": "r-1", "ocr_system": "standard", "page_range": { "end": 0, @@ -574,12 +635,17 @@ async def test_method_run_job_with_all_params(self, async_client: AsyncReducto) "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "x", + "max_share": 1, + }, "timeout": 0, }, spreadsheet={ "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 1, "split_large_tables": { "enabled": True, "size": 0, diff --git a/tests/api_resources/test_pipeline.py b/tests/api_resources/test_pipeline.py index 9bbcdeb7..40eb0388 100644 --- a/tests/api_resources/test_pipeline.py +++ b/tests/api_resources/test_pipeline.py @@ -9,10 +9,9 @@ from reducto import Reducto, AsyncReducto from tests.utils import assert_matches_type -from reducto.types import ( - PipelineResponse, - PipelineRunJobResponse, -) +from reducto.types.shared import PipelineResponse, AsyncPipelineResponse + +# pyright: reportDeprecated=false base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010") @@ -23,29 +22,34 @@ class TestPipeline: @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_method_run(self, client: Reducto) -> None: - pipeline = client.pipeline.run( - input="string", - pipeline_id="pipeline_id", - ) + with pytest.warns(DeprecationWarning): + pipeline = client.pipeline.run( + input="string", + pipeline_id="pipeline_id", + ) + assert_matches_type(PipelineResponse, pipeline, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_method_run_with_all_params(self, client: Reducto) -> None: - pipeline = client.pipeline.run( - input="string", - pipeline_id="pipeline_id", - settings={"document_password": "document_password"}, - ) + with pytest.warns(DeprecationWarning): + pipeline = client.pipeline.run( + input="string", + pipeline_id="pipeline_id", + settings={"document_password": "document_password"}, + ) + assert_matches_type(PipelineResponse, pipeline, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_raw_response_run(self, client: Reducto) -> None: - response = client.pipeline.with_raw_response.run( - input="string", - pipeline_id="pipeline_id", - ) + with pytest.warns(DeprecationWarning): + response = client.pipeline.with_raw_response.run( + input="string", + pipeline_id="pipeline_id", + ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" @@ -55,70 +59,77 @@ def test_raw_response_run(self, client: Reducto) -> None: @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_streaming_response_run(self, client: Reducto) -> None: - with client.pipeline.with_streaming_response.run( - input="string", - pipeline_id="pipeline_id", - ) as response: - assert not response.is_closed - assert response.http_request.headers.get("X-Stainless-Lang") == "python" + with pytest.warns(DeprecationWarning): + with client.pipeline.with_streaming_response.run( + input="string", + pipeline_id="pipeline_id", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" - pipeline = response.parse() - assert_matches_type(PipelineResponse, pipeline, path=["response"]) + pipeline = response.parse() + assert_matches_type(PipelineResponse, pipeline, path=["response"]) assert cast(Any, response.is_closed) is True @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_method_run_job(self, client: Reducto) -> None: - pipeline = client.pipeline.run_job( - input="string", - pipeline_id="pipeline_id", - ) - assert_matches_type(PipelineRunJobResponse, pipeline, path=["response"]) + with pytest.warns(DeprecationWarning): + pipeline = client.pipeline.run_job( + input="string", + pipeline_id="pipeline_id", + ) + + assert_matches_type(AsyncPipelineResponse, pipeline, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_method_run_job_with_all_params(self, client: Reducto) -> None: - pipeline = client.pipeline.run_job( - input="string", - pipeline_id="pipeline_id", - async_={ - "metadata": {}, - "priority": True, - "webhook": { - "channels": ["string"], - "mode": "svix", + with pytest.warns(DeprecationWarning): + pipeline = client.pipeline.run_job( + input="string", + pipeline_id="pipeline_id", + async_={ + "metadata": {}, + "priority": True, + "webhook": { + "channels": ["string"], + "mode": "svix", + }, }, - }, - settings={"document_password": "document_password"}, - ) - assert_matches_type(PipelineRunJobResponse, pipeline, path=["response"]) + settings={"document_password": "document_password"}, + ) + + assert_matches_type(AsyncPipelineResponse, pipeline, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_raw_response_run_job(self, client: Reducto) -> None: - response = client.pipeline.with_raw_response.run_job( - input="string", - pipeline_id="pipeline_id", - ) + with pytest.warns(DeprecationWarning): + response = client.pipeline.with_raw_response.run_job( + input="string", + pipeline_id="pipeline_id", + ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" pipeline = response.parse() - assert_matches_type(PipelineRunJobResponse, pipeline, path=["response"]) + assert_matches_type(AsyncPipelineResponse, pipeline, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize def test_streaming_response_run_job(self, client: Reducto) -> None: - with client.pipeline.with_streaming_response.run_job( - input="string", - pipeline_id="pipeline_id", - ) as response: - assert not response.is_closed - assert response.http_request.headers.get("X-Stainless-Lang") == "python" + with pytest.warns(DeprecationWarning): + with client.pipeline.with_streaming_response.run_job( + input="string", + pipeline_id="pipeline_id", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" - pipeline = response.parse() - assert_matches_type(PipelineRunJobResponse, pipeline, path=["response"]) + pipeline = response.parse() + assert_matches_type(AsyncPipelineResponse, pipeline, path=["response"]) assert cast(Any, response.is_closed) is True @@ -131,29 +142,34 @@ class TestAsyncPipeline: @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_method_run(self, async_client: AsyncReducto) -> None: - pipeline = await async_client.pipeline.run( - input="string", - pipeline_id="pipeline_id", - ) + with pytest.warns(DeprecationWarning): + pipeline = await async_client.pipeline.run( + input="string", + pipeline_id="pipeline_id", + ) + assert_matches_type(PipelineResponse, pipeline, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_method_run_with_all_params(self, async_client: AsyncReducto) -> None: - pipeline = await async_client.pipeline.run( - input="string", - pipeline_id="pipeline_id", - settings={"document_password": "document_password"}, - ) + with pytest.warns(DeprecationWarning): + pipeline = await async_client.pipeline.run( + input="string", + pipeline_id="pipeline_id", + settings={"document_password": "document_password"}, + ) + assert_matches_type(PipelineResponse, pipeline, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_raw_response_run(self, async_client: AsyncReducto) -> None: - response = await async_client.pipeline.with_raw_response.run( - input="string", - pipeline_id="pipeline_id", - ) + with pytest.warns(DeprecationWarning): + response = await async_client.pipeline.with_raw_response.run( + input="string", + pipeline_id="pipeline_id", + ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" @@ -163,69 +179,76 @@ async def test_raw_response_run(self, async_client: AsyncReducto) -> None: @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_streaming_response_run(self, async_client: AsyncReducto) -> None: - async with async_client.pipeline.with_streaming_response.run( - input="string", - pipeline_id="pipeline_id", - ) as response: - assert not response.is_closed - assert response.http_request.headers.get("X-Stainless-Lang") == "python" + with pytest.warns(DeprecationWarning): + async with async_client.pipeline.with_streaming_response.run( + input="string", + pipeline_id="pipeline_id", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" - pipeline = await response.parse() - assert_matches_type(PipelineResponse, pipeline, path=["response"]) + pipeline = await response.parse() + assert_matches_type(PipelineResponse, pipeline, path=["response"]) assert cast(Any, response.is_closed) is True @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_method_run_job(self, async_client: AsyncReducto) -> None: - pipeline = await async_client.pipeline.run_job( - input="string", - pipeline_id="pipeline_id", - ) - assert_matches_type(PipelineRunJobResponse, pipeline, path=["response"]) + with pytest.warns(DeprecationWarning): + pipeline = await async_client.pipeline.run_job( + input="string", + pipeline_id="pipeline_id", + ) + + assert_matches_type(AsyncPipelineResponse, pipeline, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_method_run_job_with_all_params(self, async_client: AsyncReducto) -> None: - pipeline = await async_client.pipeline.run_job( - input="string", - pipeline_id="pipeline_id", - async_={ - "metadata": {}, - "priority": True, - "webhook": { - "channels": ["string"], - "mode": "svix", + with pytest.warns(DeprecationWarning): + pipeline = await async_client.pipeline.run_job( + input="string", + pipeline_id="pipeline_id", + async_={ + "metadata": {}, + "priority": True, + "webhook": { + "channels": ["string"], + "mode": "svix", + }, }, - }, - settings={"document_password": "document_password"}, - ) - assert_matches_type(PipelineRunJobResponse, pipeline, path=["response"]) + settings={"document_password": "document_password"}, + ) + + assert_matches_type(AsyncPipelineResponse, pipeline, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_raw_response_run_job(self, async_client: AsyncReducto) -> None: - response = await async_client.pipeline.with_raw_response.run_job( - input="string", - pipeline_id="pipeline_id", - ) + with pytest.warns(DeprecationWarning): + response = await async_client.pipeline.with_raw_response.run_job( + input="string", + pipeline_id="pipeline_id", + ) assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" pipeline = await response.parse() - assert_matches_type(PipelineRunJobResponse, pipeline, path=["response"]) + assert_matches_type(AsyncPipelineResponse, pipeline, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize async def test_streaming_response_run_job(self, async_client: AsyncReducto) -> None: - async with async_client.pipeline.with_streaming_response.run_job( - input="string", - pipeline_id="pipeline_id", - ) as response: - assert not response.is_closed - assert response.http_request.headers.get("X-Stainless-Lang") == "python" - - pipeline = await response.parse() - assert_matches_type(PipelineRunJobResponse, pipeline, path=["response"]) + with pytest.warns(DeprecationWarning): + async with async_client.pipeline.with_streaming_response.run_job( + input="string", + pipeline_id="pipeline_id", + ) as response: + assert not response.is_closed + assert response.http_request.headers.get("X-Stainless-Lang") == "python" + + pipeline = await response.parse() + assert_matches_type(AsyncPipelineResponse, pipeline, path=["response"]) assert cast(Any, response.is_closed) is True diff --git a/tests/api_resources/test_split.py b/tests/api_resources/test_split.py index 408699d5..bae26cff 100644 --- a/tests/api_resources/test_split.py +++ b/tests/api_resources/test_split.py @@ -9,10 +9,7 @@ from reducto import Reducto, AsyncReducto from tests.utils import assert_matches_type -from reducto.types import ( - SplitResponse, - SplitRunJobResponse, -) +from reducto.types.shared import SplitResponse, AsyncSplitResponse base_url = os.environ.get("TEST_API_BASE_URL", "http://127.0.0.1:4010") @@ -48,9 +45,11 @@ def test_method_run_with_all_params(self, client: Reducto) -> None: ], parsing={ "enhance": { + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -75,9 +74,13 @@ def test_method_run_with_all_params(self, client: Reducto) -> None: "settings": { "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, + "model": "r-1", "ocr_system": "standard", "page_range": { "end": 0, @@ -86,19 +89,30 @@ def test_method_run_with_all_params(self, client: Reducto) -> None: "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "x", + "max_share": 1, + }, "timeout": 0, }, "spreadsheet": { "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 1, "split_large_tables": { "enabled": True, "size": 0, }, }, }, - settings={"table_cutoff": "truncate"}, + settings={ + "allow_page_overlap": True, + "auto_partition": True, + "deep_split": True, + "force_url_result": True, + "table_cutoff": "truncate", + }, split_rules="split_rules", ) assert_matches_type(SplitResponse, split, path=["response"]) @@ -153,7 +167,7 @@ def test_method_run_job(self, client: Reducto) -> None: } ], ) - assert_matches_type(SplitRunJobResponse, split, path=["response"]) + assert_matches_type(AsyncSplitResponse, split, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize @@ -177,9 +191,11 @@ def test_method_run_job_with_all_params(self, client: Reducto) -> None: }, parsing={ "enhance": { + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -204,9 +220,13 @@ def test_method_run_job_with_all_params(self, client: Reducto) -> None: "settings": { "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, + "model": "r-1", "ocr_system": "standard", "page_range": { "end": 0, @@ -215,22 +235,33 @@ def test_method_run_job_with_all_params(self, client: Reducto) -> None: "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "x", + "max_share": 1, + }, "timeout": 0, }, "spreadsheet": { "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 1, "split_large_tables": { "enabled": True, "size": 0, }, }, }, - settings={"table_cutoff": "truncate"}, + settings={ + "allow_page_overlap": True, + "auto_partition": True, + "deep_split": True, + "force_url_result": True, + "table_cutoff": "truncate", + }, split_rules="split_rules", ) - assert_matches_type(SplitRunJobResponse, split, path=["response"]) + assert_matches_type(AsyncSplitResponse, split, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize @@ -248,7 +279,7 @@ def test_raw_response_run_job(self, client: Reducto) -> None: assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" split = response.parse() - assert_matches_type(SplitRunJobResponse, split, path=["response"]) + assert_matches_type(AsyncSplitResponse, split, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize @@ -266,7 +297,7 @@ def test_streaming_response_run_job(self, client: Reducto) -> None: assert response.http_request.headers.get("X-Stainless-Lang") == "python" split = response.parse() - assert_matches_type(SplitRunJobResponse, split, path=["response"]) + assert_matches_type(AsyncSplitResponse, split, path=["response"]) assert cast(Any, response.is_closed) is True @@ -304,9 +335,11 @@ async def test_method_run_with_all_params(self, async_client: AsyncReducto) -> N ], parsing={ "enhance": { + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -331,9 +364,13 @@ async def test_method_run_with_all_params(self, async_client: AsyncReducto) -> N "settings": { "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, + "model": "r-1", "ocr_system": "standard", "page_range": { "end": 0, @@ -342,19 +379,30 @@ async def test_method_run_with_all_params(self, async_client: AsyncReducto) -> N "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "x", + "max_share": 1, + }, "timeout": 0, }, "spreadsheet": { "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 1, "split_large_tables": { "enabled": True, "size": 0, }, }, }, - settings={"table_cutoff": "truncate"}, + settings={ + "allow_page_overlap": True, + "auto_partition": True, + "deep_split": True, + "force_url_result": True, + "table_cutoff": "truncate", + }, split_rules="split_rules", ) assert_matches_type(SplitResponse, split, path=["response"]) @@ -409,7 +457,7 @@ async def test_method_run_job(self, async_client: AsyncReducto) -> None: } ], ) - assert_matches_type(SplitRunJobResponse, split, path=["response"]) + assert_matches_type(AsyncSplitResponse, split, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize @@ -433,9 +481,11 @@ async def test_method_run_job_with_all_params(self, async_client: AsyncReducto) }, parsing={ "enhance": { + "advanced_chart_agent": True, "agentic": [ { "scope": "table", + "mode": "default", "prompt": "prompt", } ], @@ -460,9 +510,13 @@ async def test_method_run_job_with_all_params(self, async_client: AsyncReducto) "settings": { "document_password": "document_password", "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 50, + "extract_document_properties": True, "extraction_mode": "ocr", "force_file_extension": "force_file_extension", "force_url_result": True, + "hybrid_vpc": {"environment": "environment"}, + "model": "r-1", "ocr_system": "standard", "page_range": { "end": 0, @@ -471,22 +525,33 @@ async def test_method_run_job_with_all_params(self, async_client: AsyncReducto) "persist_results": True, "return_images": ["figure"], "return_ocr_data": True, + "tenant_throttling": { + "tenant_id": "x", + "max_share": 1, + }, "timeout": 0, }, "spreadsheet": { "clustering": "accurate", "exclude": ["hidden_sheets"], "include": ["cell_colors"], + "max_cell_count": 1, "split_large_tables": { "enabled": True, "size": 0, }, }, }, - settings={"table_cutoff": "truncate"}, + settings={ + "allow_page_overlap": True, + "auto_partition": True, + "deep_split": True, + "force_url_result": True, + "table_cutoff": "truncate", + }, split_rules="split_rules", ) - assert_matches_type(SplitRunJobResponse, split, path=["response"]) + assert_matches_type(AsyncSplitResponse, split, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize @@ -504,7 +569,7 @@ async def test_raw_response_run_job(self, async_client: AsyncReducto) -> None: assert response.is_closed is True assert response.http_request.headers.get("X-Stainless-Lang") == "python" split = await response.parse() - assert_matches_type(SplitRunJobResponse, split, path=["response"]) + assert_matches_type(AsyncSplitResponse, split, path=["response"]) @pytest.mark.skip(reason="Mock server tests are disabled") @parametrize @@ -522,6 +587,6 @@ async def test_streaming_response_run_job(self, async_client: AsyncReducto) -> N assert response.http_request.headers.get("X-Stainless-Lang") == "python" split = await response.parse() - assert_matches_type(SplitRunJobResponse, split, path=["response"]) + assert_matches_type(AsyncSplitResponse, split, path=["response"]) assert cast(Any, response.is_closed) is True diff --git a/tests/test_client.py b/tests/test_client.py index 157a4300..95bc33e2 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -427,6 +427,30 @@ def test_default_query_option(self) -> None: client.close() + def test_hardcoded_query_params_in_url(self, client: Reducto) -> None: + request = client._build_request(FinalRequestOptions(method="get", url="/foo?beta=true")) + url = httpx.URL(request.url) + assert dict(url.params) == {"beta": "true"} + + request = client._build_request( + FinalRequestOptions( + method="get", + url="/foo?beta=true", + params={"limit": "10", "page": "abc"}, + ) + ) + url = httpx.URL(request.url) + assert dict(url.params) == {"beta": "true", "limit": "10", "page": "abc"} + + request = client._build_request( + FinalRequestOptions( + method="get", + url="/files/a%2Fb?beta=true", + params={"limit": "10"}, + ) + ) + assert request.url.raw_path == b"/files/a%2Fb?beta=true&limit=10" + def test_request_extra_json(self, client: Reducto) -> None: request = client._build_request( FinalRequestOptions( @@ -1328,6 +1352,30 @@ async def test_default_query_option(self) -> None: await client.close() + async def test_hardcoded_query_params_in_url(self, async_client: AsyncReducto) -> None: + request = async_client._build_request(FinalRequestOptions(method="get", url="/foo?beta=true")) + url = httpx.URL(request.url) + assert dict(url.params) == {"beta": "true"} + + request = async_client._build_request( + FinalRequestOptions( + method="get", + url="/foo?beta=true", + params={"limit": "10", "page": "abc"}, + ) + ) + url = httpx.URL(request.url) + assert dict(url.params) == {"beta": "true", "limit": "10", "page": "abc"} + + request = async_client._build_request( + FinalRequestOptions( + method="get", + url="/files/a%2Fb?beta=true", + params={"limit": "10"}, + ) + ) + assert request.url.raw_path == b"/files/a%2Fb?beta=true&limit=10" + def test_request_extra_json(self, client: Reducto) -> None: request = client._build_request( FinalRequestOptions( diff --git a/tests/test_deepcopy.py b/tests/test_deepcopy.py deleted file mode 100644 index f2e61051..00000000 --- a/tests/test_deepcopy.py +++ /dev/null @@ -1,58 +0,0 @@ -from reducto._utils import deepcopy_minimal - - -def assert_different_identities(obj1: object, obj2: object) -> None: - assert obj1 == obj2 - assert id(obj1) != id(obj2) - - -def test_simple_dict() -> None: - obj1 = {"foo": "bar"} - obj2 = deepcopy_minimal(obj1) - assert_different_identities(obj1, obj2) - - -def test_nested_dict() -> None: - obj1 = {"foo": {"bar": True}} - obj2 = deepcopy_minimal(obj1) - assert_different_identities(obj1, obj2) - assert_different_identities(obj1["foo"], obj2["foo"]) - - -def test_complex_nested_dict() -> None: - obj1 = {"foo": {"bar": [{"hello": "world"}]}} - obj2 = deepcopy_minimal(obj1) - assert_different_identities(obj1, obj2) - assert_different_identities(obj1["foo"], obj2["foo"]) - assert_different_identities(obj1["foo"]["bar"], obj2["foo"]["bar"]) - assert_different_identities(obj1["foo"]["bar"][0], obj2["foo"]["bar"][0]) - - -def test_simple_list() -> None: - obj1 = ["a", "b", "c"] - obj2 = deepcopy_minimal(obj1) - assert_different_identities(obj1, obj2) - - -def test_nested_list() -> None: - obj1 = ["a", [1, 2, 3]] - obj2 = deepcopy_minimal(obj1) - assert_different_identities(obj1, obj2) - assert_different_identities(obj1[1], obj2[1]) - - -class MyObject: ... - - -def test_ignores_other_types() -> None: - # custom classes - my_obj = MyObject() - obj1 = {"foo": my_obj} - obj2 = deepcopy_minimal(obj1) - assert_different_identities(obj1, obj2) - assert obj1["foo"] is my_obj - - # tuples - obj3 = ("a", "b") - obj4 = deepcopy_minimal(obj3) - assert obj3 is obj4 diff --git a/tests/test_extract_files.py b/tests/test_extract_files.py index a4c8a66f..f3cccbea 100644 --- a/tests/test_extract_files.py +++ b/tests/test_extract_files.py @@ -4,7 +4,7 @@ import pytest -from reducto._types import FileTypes +from reducto._types import FileTypes, ArrayFormat from reducto._utils import extract_files @@ -35,6 +35,12 @@ def test_multiple_files() -> None: assert query == {"documents": [{}, {}]} +def test_top_level_file_array() -> None: + query = {"files": [b"file one", b"file two"], "title": "hello"} + assert extract_files(query, paths=[["files", ""]]) == [("files[]", b"file one"), ("files[]", b"file two")] + assert query == {"title": "hello"} + + @pytest.mark.parametrize( "query,paths,expected", [ @@ -62,3 +68,24 @@ def test_ignores_incorrect_paths( expected: list[tuple[str, FileTypes]], ) -> None: assert extract_files(query, paths=paths) == expected + + +@pytest.mark.parametrize( + "array_format,expected_top_level,expected_nested", + [ + ("brackets", [("files[]", b"a"), ("files[]", b"b")], [("items[][file]", b"a"), ("items[][file]", b"b")]), + ("repeat", [("files", b"a"), ("files", b"b")], [("items[file]", b"a"), ("items[file]", b"b")]), + ("comma", [("files", b"a"), ("files", b"b")], [("items[file]", b"a"), ("items[file]", b"b")]), + ("indices", [("files[0]", b"a"), ("files[1]", b"b")], [("items[0][file]", b"a"), ("items[1][file]", b"b")]), + ], +) +def test_array_format_controls_file_field_names( + array_format: ArrayFormat, + expected_top_level: list[tuple[str, FileTypes]], + expected_nested: list[tuple[str, FileTypes]], +) -> None: + top_level = {"files": [b"a", b"b"]} + assert extract_files(top_level, paths=[["files", ""]], array_format=array_format) == expected_top_level + + nested = {"items": [{"file": b"a"}, {"file": b"b"}]} + assert extract_files(nested, paths=[["items", "", "file"]], array_format=array_format) == expected_nested diff --git a/tests/test_files.py b/tests/test_files.py index b5e73903..23535a30 100644 --- a/tests/test_files.py +++ b/tests/test_files.py @@ -4,7 +4,8 @@ import pytest from dirty_equals import IsDict, IsList, IsBytes, IsTuple -from reducto._files import to_httpx_files, async_to_httpx_files +from reducto._files import to_httpx_files, deepcopy_with_paths, async_to_httpx_files +from reducto._utils import extract_files readme_path = Path(__file__).parent.parent.joinpath("README.md") @@ -49,3 +50,99 @@ def test_string_not_allowed() -> None: "file": "foo", # type: ignore } ) + + +def assert_different_identities(obj1: object, obj2: object) -> None: + assert obj1 == obj2 + assert obj1 is not obj2 + + +class TestDeepcopyWithPaths: + def test_copies_top_level_dict(self) -> None: + original = {"file": b"data", "other": "value"} + result = deepcopy_with_paths(original, [["file"]]) + assert_different_identities(result, original) + + def test_file_value_is_same_reference(self) -> None: + file_bytes = b"contents" + original = {"file": file_bytes} + result = deepcopy_with_paths(original, [["file"]]) + assert_different_identities(result, original) + assert result["file"] is file_bytes + + def test_list_popped_wholesale(self) -> None: + files = [b"f1", b"f2"] + original = {"files": files, "title": "t"} + result = deepcopy_with_paths(original, [["files", ""]]) + assert_different_identities(result, original) + result_files = result["files"] + assert isinstance(result_files, list) + assert_different_identities(result_files, files) + + def test_nested_array_path_copies_list_and_elements(self) -> None: + elem1 = {"file": b"f1", "extra": 1} + elem2 = {"file": b"f2", "extra": 2} + original = {"items": [elem1, elem2]} + result = deepcopy_with_paths(original, [["items", "", "file"]]) + assert_different_identities(result, original) + result_items = result["items"] + assert isinstance(result_items, list) + assert_different_identities(result_items, original["items"]) + assert_different_identities(result_items[0], elem1) + assert_different_identities(result_items[1], elem2) + + def test_empty_paths_returns_same_object(self) -> None: + original = {"foo": "bar"} + result = deepcopy_with_paths(original, []) + assert result is original + + def test_multiple_paths(self) -> None: + f1 = b"file1" + f2 = b"file2" + original = {"a": f1, "b": f2, "c": "unchanged"} + result = deepcopy_with_paths(original, [["a"], ["b"]]) + assert_different_identities(result, original) + assert result["a"] is f1 + assert result["b"] is f2 + assert result["c"] is original["c"] + + def test_extract_files_does_not_mutate_original_top_level(self) -> None: + file_bytes = b"contents" + original = {"file": file_bytes, "other": "value"} + + copied = deepcopy_with_paths(original, [["file"]]) + extracted = extract_files(copied, paths=[["file"]]) + + assert extracted == [("file", file_bytes)] + assert original == {"file": file_bytes, "other": "value"} + assert copied == {"other": "value"} + + def test_extract_files_does_not_mutate_original_nested_array_path(self) -> None: + file1 = b"f1" + file2 = b"f2" + original = { + "items": [ + {"file": file1, "extra": 1}, + {"file": file2, "extra": 2}, + ], + "title": "example", + } + + copied = deepcopy_with_paths(original, [["items", "", "file"]]) + extracted = extract_files(copied, paths=[["items", "", "file"]]) + + assert [entry for _, entry in extracted] == [file1, file2] + assert original == { + "items": [ + {"file": file1, "extra": 1}, + {"file": file2, "extra": 2}, + ], + "title": "example", + } + assert copied == { + "items": [ + {"extra": 1}, + {"extra": 2}, + ], + "title": "example", + } diff --git a/tests/test_models.py b/tests/test_models.py index d14b760a..3e0e297f 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,7 +1,8 @@ import json -from typing import TYPE_CHECKING, Any, Dict, List, Union, Optional, cast +from typing import TYPE_CHECKING, Any, Dict, List, Union, Iterable, Optional, cast from datetime import datetime, timezone -from typing_extensions import Literal, Annotated, TypeAliasType +from collections import deque +from typing_extensions import Literal, Annotated, TypedDict, TypeAliasType import pytest import pydantic @@ -9,7 +10,7 @@ from reducto._utils import PropertyInfo from reducto._compat import PYDANTIC_V1, parse_obj, model_dump, model_json -from reducto._models import DISCRIMINATOR_CACHE, BaseModel, construct_type +from reducto._models import DISCRIMINATOR_CACHE, BaseModel, EagerIterable, construct_type class BasicModel(BaseModel): @@ -961,3 +962,56 @@ def __getattr__(self, attr: str) -> Item: ... assert model.a.prop == 1 assert isinstance(model.a, Item) assert model.other == "foo" + + +# NOTE: Workaround for Pydantic Iterable behavior. +# Iterable fields are replaced with a ValidatorIterator and may be consumed +# during serialization, which can cause subsequent dumps to return empty data. +# See: https://github.com/pydantic/pydantic/issues/9541 +@pytest.mark.parametrize( + "data, expected_validated", + [ + ([1, 2, 3], [1, 2, 3]), + ((1, 2, 3), (1, 2, 3)), + (set([1, 2, 3]), set([1, 2, 3])), + (iter([1, 2, 3]), [1, 2, 3]), + ([], []), + ((x for x in [1, 2, 3]), [1, 2, 3]), + (map(lambda x: x, [1, 2, 3]), [1, 2, 3]), + (frozenset([1, 2, 3]), frozenset([1, 2, 3])), + (deque([1, 2, 3]), deque([1, 2, 3])), + ], + ids=["list", "tuple", "set", "iterator", "empty", "generator", "map", "frozenset", "deque"], +) +@pytest.mark.skipif(PYDANTIC_V1, reason="this is only supported in pydantic v2") +def test_iterable_construction(data: Iterable[int], expected_validated: Iterable[int]) -> None: + class TypeWithIterable(TypedDict): + items: EagerIterable[int] + + class Model(BaseModel): + data: TypeWithIterable + + m = Model.model_validate({"data": {"items": data}}) + assert m.data["items"] == expected_validated + + # Verify repeated dumps don't lose data (the original bug) + assert m.model_dump()["data"]["items"] == list(expected_validated) + assert m.model_dump()["data"]["items"] == list(expected_validated) + + +@pytest.mark.skipif(PYDANTIC_V1, reason="this is only supported in pydantic v2") +def test_iterable_construction_str_falls_back_to_list() -> None: + # str is iterable (over chars), but str(list_of_chars) produces the list's repr + # rather than reconstructing a string from items. We special-case str to fall + # back to list instead of attempting reconstruction. + class TypeWithIterable(TypedDict): + items: EagerIterable[str] + + class Model(BaseModel): + data: TypeWithIterable + + m = Model.model_validate({"data": {"items": "hello"}}) + + # falls back to list of chars rather than calling str(["h", "e", "l", "l", "o"]) + assert m.data["items"] == ["h", "e", "l", "l", "o"] + assert m.model_dump()["data"]["items"] == ["h", "e", "l", "l", "o"]