diff --git a/.eslintrc.json b/.eslintrc.json index c433d09..07ee36e 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -5,7 +5,6 @@ }, "extends": [ "airbnb-base", - "prettier/@typescript-eslint", "plugin:prettier/recommended" ], "globals": { diff --git a/.gitignore b/.gitignore index a42bca8..8df2bc6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .vscode .idea -node_modules \ No newline at end of file +node_modules +exhaustive-test.js \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 365ce7c..61b5309 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,21 @@ # Changelog -## [1.8.3](https://github.com/ScrapingBee/scrapingbee-node/compare/v1.8.2...v1.8.3) (2026-06-30) +## [2.0.0](https://github.com/ScrapingBee/scrapingbee-node/compare/v1.8.2...v2.0.0) (2026-08-05) ### Features -- Added Auto-Mode support for the HTML API: pass `mode: 'auto'` (GET only) to let ScrapingBee pick the cheapest scraping config that succeeds, charged only for the winning config. Optional `max_cost` (integer ≥ 1) caps the credits a request may cost. The credits charged are returned in the `Spb-auto-cost` response header. -- Added `mode` and `max_cost` to the `HtmlApiParams` type for autocomplete. +- Added `fastSearch()` method for Fast Search API +- Added `amazonPricing()` method for Amazon Pricing API +- Added `gemini()` method for Gemini API +- Added `youtubeSubtitles()` method for YouTube Subtitles API (replaces the removed `youtubeTranscript()`) +- Expanded param type hints for all endpoints to match the documented API contract (e.g. HTML API `mode`/`max_cost`, Google `pages`/`date_range`/geo/`sort_by`/price filters, Amazon Search `autoselect_variant`, Walmart Search `start_page`) +- Added `tag` optional param (request label) to every endpoint's params type except Usage, including a new `YouTubeMetadataParams` type so `youtubeMetadata()` now accepts a `params` object + +### Removed + +- Removed deprecated `get()` and `post()` methods (use `htmlApi()` with `method: 'GET'`/`'POST'` instead) +- Removed `youtubeTranscript()` method (endpoint no longer supported; use `youtubeSubtitles()` instead) +- Removed `youtubeTrainability()` method (no longer supported) ## [1.8.2](https://github.com/ScrapingBee/scrapingbee-node/compare/v1.8.0...v1.8.2) (2026-01-22) diff --git a/README.md b/README.md index 33d63fd..44fa1b0 100644 --- a/README.md +++ b/README.md @@ -20,10 +20,12 @@ Signup to ScrapingBee to [get your API key](https://app.scrapingbee.com/account/ - [HTML API](#html-api) - [Google Search API](#google-search-api) +- [Fast Search API](#fast-search-api) - [Amazon API](#amazon-api) - [Walmart API](#walmart-api) - [YouTube API](#youtube-api) - [ChatGPT API](#chatgpt-api) +- [Gemini API](#gemini-api) - [Usage API](#usage-api) --- @@ -110,7 +112,7 @@ async function post(url) { const response = await client.htmlApi({ url: url, method: 'POST', - 'username=user&password=pass', + data: 'username=user&password=pass', params: { render_js: false, }, @@ -127,7 +129,7 @@ async function post(url) { console.log(text); } -post('https://httpbin.org/post'); +post('https://httpbin.scrapingbee.com/post'); ``` ### Screenshot @@ -186,6 +188,33 @@ googleSearch('web scraping tools'); --- +## Fast Search API + +Retrieve a lightweight Google result set optimized for sub-second responses. + +```javascript +const { ScrapingBeeClient } = require('scrapingbee'); + +async function fastSearch(query) { + const client = new ScrapingBeeClient('YOUR-API-KEY'); + const response = await client.fastSearch({ + search: query, + params: { + page: 1, + country_code: 'us', + language: 'en', + tag: 'my-request', + } + }); + + console.log(response.data); +} + +fastSearch('web scraping tools'); +``` + +--- + ## Amazon API Scrape Amazon search results and product details. @@ -245,6 +274,32 @@ async function amazonProduct(asin) { amazonProduct('B0D2Q9397Y'); ``` +### Amazon Pricing + +```javascript +const { ScrapingBeeClient } = require('scrapingbee'); + +async function amazonPricing(asin) { + const client = new ScrapingBeeClient('YOUR-API-KEY'); + const response = await client.amazonPricing({ + asin: asin, + params: { + domain: 'com', + language: 'en', + zip_code: '10001', + currency: 'USD', + device: 'desktop', + light_request: true, + add_html: false, + } + }); + + console.log(response.data); +} + +amazonPricing('B0D2Q9397Y'); +``` + --- ## Walmart API @@ -307,7 +362,7 @@ walmartProduct('123456789'); ## YouTube API -Scrape YouTube search results, video metadata, transcripts, and trainability data. +Scrape YouTube search results, video metadata, and subtitles. ### YouTube Search @@ -353,59 +408,67 @@ async function youtubeMetadata(videoId) { youtubeMetadata('dQw4w9WgXcQ'); ``` -### YouTube Transcript +### YouTube Subtitles ```javascript const { ScrapingBeeClient } = require('scrapingbee'); -async function youtubeTranscript(videoId) { +async function youtubeSubtitles(videoId) { const client = new ScrapingBeeClient('YOUR-API-KEY'); - const response = await client.youtubeTranscript({ + const response = await client.youtubeSubtitles({ video_id: videoId, params: { language: 'en', - transcript_origin: 'auto_generated', + subtitle_origin: 'auto_generated', } }); console.log(response.data); } -youtubeTranscript('dQw4w9WgXcQ'); +youtubeSubtitles('dQw4w9WgXcQ'); ``` -### YouTube Trainability +--- + +## ChatGPT API + +Use ChatGPT with optional web search capabilities. ```javascript const { ScrapingBeeClient } = require('scrapingbee'); -async function youtubeTrainability(videoId) { +async function askChatGPT(prompt) { const client = new ScrapingBeeClient('YOUR-API-KEY'); - const response = await client.youtubeTrainability({ - video_id: videoId, + const response = await client.chatGPT({ + prompt: prompt, + params: { + search: true, + country_code: 'us', + add_html: false, + } }); console.log(response.data); } -youtubeTrainability('dQw4w9WgXcQ'); +askChatGPT('What are the latest web scraping trends?'); ``` --- -## ChatGPT API +## Gemini API -Use ChatGPT with optional web search capabilities. +Ask Gemini through ScrapingBee and receive citation objects when available. ```javascript const { ScrapingBeeClient } = require('scrapingbee'); -async function askChatGPT(prompt) { +async function askGemini(prompt) { const client = new ScrapingBeeClient('YOUR-API-KEY'); - const response = await client.chatGPT({ + const response = await client.gemini({ prompt: prompt, params: { - search: true, country_code: 'us', add_html: false, } @@ -414,7 +477,7 @@ async function askChatGPT(prompt) { console.log(response.data); } -askChatGPT('What are the latest web scraping trends?'); +askGemini('What are the latest web scraping trends?'); ``` --- @@ -475,22 +538,6 @@ client.googleSearch({ search: 'test' }) --- -## Legacy Methods (Deprecated) - -The `get()` and `post()` methods are deprecated and will be removed in a future version. Please use `htmlApi()` instead. - -```javascript -// Deprecated -await client.get({ url: '...' }); -await client.post({ url: '...' }); - -// Use instead -await client.htmlApi({ url: '...', method: 'GET' }); -await client.htmlApi({ url: '...', method: 'POST' }); -``` - ---- - ## Documentation For more details on all available parameters, visit [ScrapingBee's documentation](https://www.scrapingbee.com/documentation/). \ No newline at end of file diff --git a/dist/index.d.ts b/dist/index.d.ts index 9b78b2c..c2e0b9e 100644 --- a/dist/index.d.ts +++ b/dist/index.d.ts @@ -1,5 +1,6 @@ import { AxiosPromise } from 'axios'; export declare type HtmlApiParams = { + tag?: string; ai_extract_rules?: object | string; ai_query?: string; ai_selector?: string; @@ -49,15 +50,24 @@ export interface HtmlApiConfig { timeout?: number; } export declare type GoogleSearchParams = { + tag?: string; add_html?: boolean; country_code?: string; + date_range?: string; device?: string; extra_params?: string; language?: string; + latitude?: number; light_request?: boolean; + longitude?: number; + max_price?: number; + min_price?: number; nfpr?: boolean; page?: number; + pages?: number; + radius?: number; search_type?: string; + sort_by?: string; } & { [key: string]: any; }; @@ -68,7 +78,9 @@ export interface GoogleSearchConfig { timeout?: number; } export declare type AmazonSearchParams = { + tag?: string; add_html?: boolean; + autoselect_variant?: boolean; category_id?: string; country?: string; currency?: string; @@ -92,6 +104,7 @@ export interface AmazonSearchConfig { timeout?: number; } export declare type AmazonProductParams = { + tag?: string; add_html?: boolean; autoselect_variant?: boolean; country?: string; @@ -112,6 +125,7 @@ export interface AmazonProductConfig { timeout?: number; } export declare type WalmartSearchParams = { + tag?: string; add_html?: boolean; delivery_zip?: string; device?: string; @@ -123,6 +137,7 @@ export declare type WalmartSearchParams = { min_price?: number; screenshot?: boolean; sort_by?: string; + start_page?: number; store_id?: string; } & { [key: string]: any; @@ -134,6 +149,7 @@ export interface WalmartSearchConfig { timeout?: number; } export declare type WalmartProductParams = { + tag?: string; add_html?: boolean; delivery_zip?: string; device?: string; @@ -151,6 +167,7 @@ export interface WalmartProductConfig { timeout?: number; } export declare type ChatGPTParams = { + tag?: string; add_html?: boolean; country_code?: string; search?: boolean; @@ -164,6 +181,7 @@ export interface ChatGPTConfig { timeout?: number; } export declare type YouTubeSearchParams = { + tag?: string; '360'?: boolean; '3d'?: boolean; '4k'?: boolean; @@ -188,25 +206,73 @@ export interface YouTubeSearchConfig { retries?: number; timeout?: number; } +export declare type YouTubeMetadataParams = { + tag?: string; +} & { + [key: string]: any; +}; export interface YouTubeMetadataConfig { video_id: string; + params?: YouTubeMetadataParams; retries?: number; timeout?: number; } -export declare type YouTubeTranscriptParams = { +export declare type YouTubeSubtitlesParams = { + tag?: string; language?: string; - transcript_origin?: string; + subtitle_origin?: string; } & { [key: string]: any; }; -export interface YouTubeTranscriptConfig { +export interface YouTubeSubtitlesConfig { video_id: string; - params?: YouTubeTranscriptParams; + params?: YouTubeSubtitlesParams; retries?: number; timeout?: number; } -export interface YouTubeTrainabilityConfig { - video_id: string; +export declare type FastSearchParams = { + country_code?: string; + language?: string; + page?: number; + tag?: string; +} & { + [key: string]: any; +}; +export interface FastSearchConfig { + search: string; + params?: FastSearchParams; + retries?: number; + timeout?: number; +} +export declare type AmazonPricingParams = { + tag?: string; + add_html?: boolean; + country?: string; + currency?: string; + device?: string; + domain?: string; + language?: string; + light_request?: boolean; + zip_code?: string; +} & { + [key: string]: any; +}; +export interface AmazonPricingConfig { + asin: string; + params?: AmazonPricingParams; + retries?: number; + timeout?: number; +} +export declare type GeminiParams = { + tag?: string; + add_html?: boolean; + country_code?: string; +} & { + [key: string]: any; +}; +export interface GeminiConfig { + prompt: string; + params?: GeminiParams; retries?: number; timeout?: number; } @@ -218,14 +284,6 @@ export declare class ScrapingBeeClient { readonly api_key: string; constructor(api_key: string); private request; - /** - * @deprecated Use htmlApi() instead. This method will be removed in version 2.0.0. - */ - get: (config: HtmlApiConfig) => AxiosPromise; - /** - * @deprecated Use htmlApi() instead. This method will be removed in version 2.0.0. - */ - post: (config: HtmlApiConfig) => AxiosPromise; googleSearch(config: GoogleSearchConfig): AxiosPromise; amazonSearch(config: AmazonSearchConfig): AxiosPromise; amazonProduct(config: AmazonProductConfig): AxiosPromise; @@ -234,8 +292,10 @@ export declare class ScrapingBeeClient { chatGPT(config: ChatGPTConfig): AxiosPromise; youtubeSearch(config: YouTubeSearchConfig): AxiosPromise; youtubeMetadata(config: YouTubeMetadataConfig): AxiosPromise; - youtubeTranscript(config: YouTubeTranscriptConfig): AxiosPromise; - youtubeTrainability(config: YouTubeTrainabilityConfig): AxiosPromise; + youtubeSubtitles(config: YouTubeSubtitlesConfig): AxiosPromise; + fastSearch(config: FastSearchConfig): AxiosPromise; + amazonPricing(config: AmazonPricingConfig): AxiosPromise; + gemini(config: GeminiConfig): AxiosPromise; htmlApi(config: HtmlApiConfig): AxiosPromise; usage(config?: UsageConfig): AxiosPromise; } diff --git a/dist/index.js b/dist/index.js index d5d122e..9cfeb43 100644 --- a/dist/index.js +++ b/dist/index.js @@ -5,7 +5,6 @@ var __importDefault = (this && this.__importDefault) || function (mod) { Object.defineProperty(exports, "__esModule", { value: true }); exports.ScrapingBeeClient = void 0; const axios_1 = __importDefault(require("axios")); -const util_1 = require("util"); const axios_retry_1 = __importDefault(require("axios-retry")); const utils_1 = require("./utils"); const HTML_API_URL = 'https://app.scrapingbee.com/api/v1/'; @@ -17,51 +16,13 @@ const WALMART_PRODUCT_API_URL = 'https://app.scrapingbee.com/api/v1/walmart/prod const CHATGPT_API_URL = 'https://app.scrapingbee.com/api/v1/chatgpt'; const YOUTUBE_SEARCH_API_URL = 'https://app.scrapingbee.com/api/v1/youtube/search'; const YOUTUBE_METADATA_API_URL = 'https://app.scrapingbee.com/api/v1/youtube/metadata'; -const YOUTUBE_TRANSCRIPT_API_URL = 'https://app.scrapingbee.com/api/v1/youtube/transcript'; -const YOUTUBE_TRAINABILITY_API_URL = 'https://app.scrapingbee.com/api/v1/youtube/trainability'; +const YOUTUBE_SUBTITLES_API_URL = 'https://app.scrapingbee.com/api/v1/youtube/subtitles'; +const FAST_SEARCH_API_URL = 'https://app.scrapingbee.com/api/v1/fast_search'; +const AMAZON_PRICING_API_URL = 'https://app.scrapingbee.com/api/v1/amazon/pricing'; +const GEMINI_API_URL = 'https://app.scrapingbee.com/api/v1/gemini'; const USAGE_API_URL = 'https://app.scrapingbee.com/api/v1/usage'; class ScrapingBeeClient { constructor(api_key) { - /** - * @deprecated Use htmlApi() instead. This method will be removed in version 2.0.0. - */ - this.get = util_1.deprecate((config) => { - var _a; - let params = Object.assign(Object.assign({}, config.params), { url: config.url, cookies: config.cookies }); - let headers = utils_1.process_headers(config.headers); - if (Object.keys((_a = config.headers) !== null && _a !== void 0 ? _a : {}).length > 0) { - params.forward_headers = true; - } - return this.request({ - method: 'GET', - endpoint: HTML_API_URL, - params: utils_1.process_params(params), - headers: headers, - data: config.data, - retries: config.retries, - timeout: config.timeout, - }); - }, 'ScrapingBeeClient.get() is deprecated. Please use client.htmlApi() instead. This method will be removed in version 2.0.0.'); - /** - * @deprecated Use htmlApi() instead. This method will be removed in version 2.0.0. - */ - this.post = util_1.deprecate((config) => { - var _a; - let params = Object.assign(Object.assign({}, config.params), { url: config.url, cookies: config.cookies }); - let headers = utils_1.process_headers(config.headers); - if (Object.keys((_a = config.headers) !== null && _a !== void 0 ? _a : {}).length > 0) { - params.forward_headers = true; - } - return this.request({ - method: 'POST', - endpoint: HTML_API_URL, - params: utils_1.process_params(params), - headers: headers, - data: config.data, - retries: config.retries, - timeout: config.timeout, - }); - }, 'ScrapingBeeClient.post() is deprecated. Please use client.htmlApi() instead. This method will be removed in version 2.0.0.'); this.api_key = api_key; } request(config) { @@ -153,9 +114,7 @@ class ScrapingBeeClient { }); } youtubeMetadata(config) { - const params = { - video_id: config.video_id, - }; + const params = Object.assign({ video_id: config.video_id }, config.params); return this.request({ method: 'GET', endpoint: YOUTUBE_METADATA_API_URL, @@ -164,23 +123,41 @@ class ScrapingBeeClient { timeout: config.timeout, }); } - youtubeTranscript(config) { + youtubeSubtitles(config) { const params = Object.assign({ video_id: config.video_id }, config.params); return this.request({ method: 'GET', - endpoint: YOUTUBE_TRANSCRIPT_API_URL, + endpoint: YOUTUBE_SUBTITLES_API_URL, params, retries: config.retries, timeout: config.timeout, }); } - youtubeTrainability(config) { - const params = { - video_id: config.video_id, - }; + fastSearch(config) { + const params = Object.assign({ search: config.search }, config.params); + return this.request({ + method: 'GET', + endpoint: FAST_SEARCH_API_URL, + params, + retries: config.retries, + timeout: config.timeout, + }); + } + amazonPricing(config) { + const params = Object.assign({ asin: config.asin }, config.params); + return this.request({ + method: 'GET', + endpoint: AMAZON_PRICING_API_URL, + params, + retries: config.retries, + timeout: config.timeout, + }); + } + gemini(config) { + const params = Object.assign({ prompt: config.prompt }, config.params); return this.request({ method: 'GET', - endpoint: YOUTUBE_TRAINABILITY_API_URL, + endpoint: GEMINI_API_URL, params, retries: config.retries, timeout: config.timeout, diff --git a/dist/version.d.ts b/dist/version.d.ts index 69c1342..8ad8e44 100644 --- a/dist/version.d.ts +++ b/dist/version.d.ts @@ -1 +1 @@ -export declare const LIB_VERSION = "1.8.3"; +export declare const LIB_VERSION = "2.0.0"; diff --git a/dist/version.js b/dist/version.js index 502968c..61d1462 100644 --- a/dist/version.js +++ b/dist/version.js @@ -1,4 +1,4 @@ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); exports.LIB_VERSION = void 0; -exports.LIB_VERSION = "1.8.3"; +exports.LIB_VERSION = "2.0.0"; diff --git a/manual-test.js b/manual-test.js index 9903b64..01db9ea 100644 --- a/manual-test.js +++ b/manual-test.js @@ -21,59 +21,14 @@ function assert(condition, message) { } // ============================================ -// Legacy HTML API Tests -// ============================================ - -async function testHtmlGet() { - console.log('=== Testing HTML API - GET ==='); - try { - const response = await client.get({ - url: 'https://httpbin.org/get', - params: { render_js: false } - }); - - assert(response.status === 200, `Expected status 200, got ${response.status}`); - assert(response.data, 'Response data is empty'); - assert(response.data.toString().includes('httpbin'), 'Response does not contain expected content'); - - console.log('Status:', response.status); - console.log('✅ HTML GET test passed!\n'); - } catch (error) { - console.log('❌ HTML GET test failed:', error.message); - throw error; - } -} - -async function testHtmlPost() { - console.log('=== Testing HTML API - POST ==='); - try { - const response = await client.post({ - url: 'https://httpbin.org/post', - params: { render_js: false }, - data: 'test=data' - }); - - assert(response.status === 200, `Expected status 200, got ${response.status}`); - assert(response.data, 'Response data is empty'); - assert(response.data.toString().includes('test'), 'Response does not contain posted data'); - - console.log('Status:', response.status); - console.log('✅ HTML POST test passed!\n'); - } catch (error) { - console.log('❌ HTML POST test failed:', error.message); - throw error; - } -} - -// ============================================ -// New HTML API Tests +// HTML API Tests // ============================================ async function testHtmlApiGet() { console.log('=== Testing HTML API (New) - GET ==='); try { const response = await client.htmlApi({ - url: 'https://httpbin.org/get', + url: 'https://httpbin.scrapingbee.com/get', method: 'GET', params: { render_js: false } }); @@ -94,7 +49,7 @@ async function testHtmlApiPost() { console.log('=== Testing HTML API (New) - POST ==='); try { const response = await client.htmlApi({ - url: 'https://httpbin.org/post', + url: 'https://httpbin.scrapingbee.com/post', method: 'POST', params: { render_js: false }, data: 'test=data' @@ -213,7 +168,7 @@ async function testHtmlApiJsonResponse() { console.log('=== Testing HTML API - JSON Response ==='); try { const response = await client.htmlApi({ - url: 'https://httpbin.org/get', + url: 'https://httpbin.scrapingbee.com/get', params: { render_js: false, json_response: true @@ -243,7 +198,7 @@ async function testHtmlApiWithHeaders() { console.log('=== Testing HTML API - Custom Headers ==='); try { const response = await client.htmlApi({ - url: 'https://httpbin.org/headers', + url: 'https://httpbin.scrapingbee.com/headers', params: { render_js: false }, headers: { 'X-Custom-Header': 'CustomValue123' @@ -265,7 +220,7 @@ async function testHtmlApiWithCookies() { console.log('=== Testing HTML API - Custom Cookies ==='); try { const response = await client.htmlApi({ - url: 'https://httpbin.org/cookies', + url: 'https://httpbin.scrapingbee.com/cookies', params: { render_js: false }, cookies: { session_id: 'abc123', @@ -290,7 +245,7 @@ async function testHtmlApiPostWithHeadersAndCookies() { console.log('=== Testing HTML API - POST with Headers + Cookies ==='); try { const response = await client.htmlApi({ - url: 'https://httpbin.org/post', + url: 'https://httpbin.scrapingbee.com/post', method: 'POST', params: { render_js: false }, headers: { 'X-Test-Header': 'TestValue' }, @@ -337,6 +292,30 @@ async function testGoogleSearch() { } } +async function testFastSearch() { + console.log('=== Testing Fast Search API ==='); + try { + const response = await client.fastSearch({ + search: 'scrapingbee', + params: { language: 'en', country_code: 'us', page: 1, tag: 'fast-search-test' } + }); + + assert(response.status === 200, `Expected status 200, got ${response.status}`); + + const data = parseResponse(response); + assert(data.organic, 'Missing organic results in response'); + assert(Array.isArray(data.organic), 'organic results is not an array'); + assert(data.organic.length > 0, 'No organic results found'); + + console.log('Status:', response.status); + console.log('Results found:', data.organic.length); + console.log('✅ Fast Search test passed!\n'); + } catch (error) { + console.log('❌ Fast Search test failed:', error.message); + throw error; + } +} + // ============================================ // Amazon API // ============================================ @@ -387,6 +366,33 @@ async function testAmazonProduct() { } } +async function testAmazonPricing() { + console.log('=== Testing Amazon Pricing API ==='); + try { + // NOTE: per docs, do not send `country` matching the domain's country + // (e.g. country=us & domain=com returns 400). Use zip_code for localization. + const response = await client.amazonPricing({ + asin: 'B0D2Q9397Y', + params: { domain: 'com', zip_code: '10001', light_request: true } + }); + + assert(response.status === 200, `Expected status 200, got ${response.status}`); + + const data = parseResponse(response); + assert(data.asin, 'Missing asin in response'); + assert(data.pricing, 'Missing pricing in response'); + assert(Array.isArray(data.pricing), 'pricing is not an array'); + assert(data.pricing.length > 0, 'No pricing offers found'); + + console.log('Status:', response.status); + console.log('Offers found:', data.pricing.length); + console.log('✅ Amazon Pricing test passed!\n'); + } catch (error) { + console.log('❌ Amazon Pricing test failed:', error.message); + throw error; + } +} + // ============================================ // Walmart API // ============================================ @@ -463,6 +469,28 @@ async function testChatGPT() { } } +async function testGemini() { + console.log('=== Testing Gemini API ==='); + try { + const response = await client.gemini({ + prompt: 'What is web scraping? Answer in one sentence.', + params: { country_code: 'us' } + }); + + assert(response.status === 200, `Expected status 200, got ${response.status}`); + + const data = parseResponse(response); + assert(data.results_text || data.results_markdown, 'Missing response text'); + + console.log('Status:', response.status); + console.log('Response:', (data.results_text || data.results_markdown).substring(0, 100)); + console.log('✅ Gemini test passed!\n'); + } catch (error) { + console.log('❌ Gemini test failed:', error.message); + throw error; + } +} + // ============================================ // YouTube API // ============================================ @@ -479,11 +507,11 @@ async function testYouTubeSearch() { const data = parseResponse(response); assert(data.results, 'Missing results in response'); - assert(Array.isArray(data.results), 'results is not an array'); + assert(typeof data.results === 'string', 'results is not a string'); assert(data.results.length > 0, 'No results found'); console.log('Status:', response.status); - console.log('Results found:', data.results.length); + console.log('Results length:', data.results.length); console.log('✅ YouTube Search test passed!\n'); } catch (error) { console.log('❌ YouTube Search test failed:', error.message); @@ -495,7 +523,8 @@ async function testYouTubeMetadata() { console.log('=== Testing YouTube Metadata API ==='); try { const response = await client.youtubeMetadata({ - video_id: 'dQw4w9WgXcQ' + video_id: 'dQw4w9WgXcQ', + params: { tag: 'metadata-test' } }); assert(response.status === 200, `Expected status 200, got ${response.status}`); @@ -512,45 +541,24 @@ async function testYouTubeMetadata() { } } -async function testYouTubeTranscript() { - console.log('=== Testing YouTube Transcript API ==='); +async function testYouTubeSubtitles() { + console.log('=== Testing YouTube Subtitles API ==='); try { - const response = await client.youtubeTranscript({ + const response = await client.youtubeSubtitles({ video_id: 'sfyL4BswUeE', - params: { language: 'en' } - }); - - assert(response.status === 200, `Expected status 200, got ${response.status}`); - - const data = parseResponse(response); - assert(data.text || data.transcript, 'Missing transcript in response'); - - console.log('Status:', response.status); - console.log('Transcript preview:', (data.text || JSON.stringify(data.transcript)).substring(0, 100)); - console.log('✅ YouTube Transcript test passed!\n'); - } catch (error) { - console.log('❌ YouTube Transcript test failed:', error.message); - throw error; - } -} - -async function testYouTubeTrainability() { - console.log('=== Testing YouTube Trainability API ==='); - try { - const response = await client.youtubeTrainability({ - video_id: 'dQw4w9WgXcQ' + params: { language: 'en', subtitle_origin: 'auto_generated' } }); assert(response.status === 200, `Expected status 200, got ${response.status}`); const data = parseResponse(response); - assert(data.permitted !== undefined, 'Missing permitted field in response'); + assert(data.subtitles, 'Missing subtitles in response'); console.log('Status:', response.status); - console.log('Permitted:', data.permitted); - console.log('✅ YouTube Trainability test passed!\n'); + console.log('Subtitles preview:', JSON.stringify(data.subtitles).substring(0, 100)); + console.log('✅ YouTube Subtitles test passed!\n'); } catch (error) { - console.log('❌ YouTube Trainability test failed:', error.message); + console.log('❌ YouTube Subtitles test failed:', error.message); throw error; } } @@ -593,11 +601,7 @@ async function runTests() { let failed = 0; const tests = [ - // Legacy HTML API - testHtmlGet, - testHtmlPost, - - // New HTML API + // HTML API testHtmlApiGet, testHtmlApiPost, testHtmlApiExtractRules, @@ -610,15 +614,17 @@ async function runTests() { // Other APIs testGoogleSearch, + testFastSearch, testAmazonSearch, testAmazonProduct, + testAmazonPricing, testWalmartSearch, testWalmartProduct, testChatGPT, + testGemini, testYouTubeSearch, testYouTubeMetadata, - testYouTubeTranscript, - testYouTubeTrainability, + testYouTubeSubtitles, testUsage, ]; diff --git a/package.json b/package.json index 096e354..bd03001 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "scrapingbee", - "version": "1.8.3", + "version": "2.0.0", "description": "ScrapingBee Node SDK", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/src/index.ts b/src/index.ts index b26be22..4991e2f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,4 @@ import axios, { AxiosPromise, AxiosRequestConfig, Method } from 'axios'; -import { deprecate } from 'util'; import axiosRetry from 'axios-retry'; import { process_params, process_headers } from './utils'; @@ -13,13 +12,16 @@ const WALMART_PRODUCT_API_URL: string = 'https://app.scrapingbee.com/api/v1/walm const CHATGPT_API_URL: string = 'https://app.scrapingbee.com/api/v1/chatgpt'; const YOUTUBE_SEARCH_API_URL: string = 'https://app.scrapingbee.com/api/v1/youtube/search'; const YOUTUBE_METADATA_API_URL: string = 'https://app.scrapingbee.com/api/v1/youtube/metadata'; -const YOUTUBE_TRANSCRIPT_API_URL: string = 'https://app.scrapingbee.com/api/v1/youtube/transcript'; -const YOUTUBE_TRAINABILITY_API_URL: string = 'https://app.scrapingbee.com/api/v1/youtube/trainability'; +const YOUTUBE_SUBTITLES_API_URL: string = 'https://app.scrapingbee.com/api/v1/youtube/subtitles'; +const FAST_SEARCH_API_URL: string = 'https://app.scrapingbee.com/api/v1/fast_search'; +const AMAZON_PRICING_API_URL: string = 'https://app.scrapingbee.com/api/v1/amazon/pricing'; +const GEMINI_API_URL: string = 'https://app.scrapingbee.com/api/v1/gemini'; const USAGE_API_URL: string = 'https://app.scrapingbee.com/api/v1/usage'; // HTML API export type HtmlApiParams = { + tag?: string; ai_extract_rules?: object | string; ai_query?: string; ai_selector?: string; @@ -73,15 +75,24 @@ export interface HtmlApiConfig { // GOOGLE export type GoogleSearchParams = { + tag?: string; add_html?: boolean; country_code?: string; + date_range?: string; device?: string; extra_params?: string; language?: string; + latitude?: number; light_request?: boolean; + longitude?: number; + max_price?: number; + min_price?: number; nfpr?: boolean; page?: number; + pages?: number; + radius?: number; search_type?: string; + sort_by?: string; } & { [key: string]: any; }; @@ -96,7 +107,9 @@ export interface GoogleSearchConfig { // AMAZON export type AmazonSearchParams = { + tag?: string; add_html?: boolean; + autoselect_variant?: boolean; category_id?: string; country?: string; currency?: string; @@ -122,6 +135,7 @@ export interface AmazonSearchConfig { } export type AmazonProductParams = { + tag?: string; add_html?: boolean; autoselect_variant?: boolean; country?: string; @@ -146,6 +160,7 @@ export interface AmazonProductConfig { // WALMART export type WalmartSearchParams = { + tag?: string; add_html?: boolean; delivery_zip?: string; device?: string; @@ -157,6 +172,7 @@ export type WalmartSearchParams = { min_price?: number; screenshot?: boolean; sort_by?: string; + start_page?: number; store_id?: string; } & { [key: string]: any; @@ -170,6 +186,7 @@ export interface WalmartSearchConfig { } export type WalmartProductParams = { + tag?: string; add_html?: boolean; delivery_zip?: string; device?: string; @@ -191,6 +208,7 @@ export interface WalmartProductConfig { // CHATGPT export type ChatGPTParams = { + tag?: string; add_html?: boolean; country_code?: string; search?: boolean; @@ -208,6 +226,7 @@ export interface ChatGPTConfig { // YOUTUBE export type YouTubeSearchParams = { + tag?: string; '360'?: boolean; '3d'?: boolean; '4k'?: boolean; @@ -234,28 +253,88 @@ export interface YouTubeSearchConfig { timeout?: number; } +export type YouTubeMetadataParams = { + tag?: string; +} & { + [key: string]: any; +}; + export interface YouTubeMetadataConfig { video_id: string; + params?: YouTubeMetadataParams; retries?: number; timeout?: number; } -export type YouTubeTranscriptParams = { +export type YouTubeSubtitlesParams = { + tag?: string; language?: string; - transcript_origin?: string; + subtitle_origin?: string; } & { [key: string]: any; }; -export interface YouTubeTranscriptConfig { +export interface YouTubeSubtitlesConfig { video_id: string; - params?: YouTubeTranscriptParams; + params?: YouTubeSubtitlesParams; retries?: number; timeout?: number; } -export interface YouTubeTrainabilityConfig { - video_id: string; +// FAST SEARCH + +export type FastSearchParams = { + country_code?: string; + language?: string; + page?: number; + tag?: string; +} & { + [key: string]: any; +}; + +export interface FastSearchConfig { + search: string; + params?: FastSearchParams; + retries?: number; + timeout?: number; +} + +// AMAZON PRICING + +export type AmazonPricingParams = { + tag?: string; + add_html?: boolean; + country?: string; + currency?: string; + device?: string; + domain?: string; + language?: string; + light_request?: boolean; + zip_code?: string; +} & { + [key: string]: any; +}; + +export interface AmazonPricingConfig { + asin: string; + params?: AmazonPricingParams; + retries?: number; + timeout?: number; +} + +// GEMINI + +export type GeminiParams = { + tag?: string; + add_html?: boolean; + country_code?: string; +} & { + [key: string]: any; +}; + +export interface GeminiConfig { + prompt: string; + params?: GeminiParams; retries?: number; timeout?: number; } @@ -296,58 +375,6 @@ export class ScrapingBeeClient { return axios(axiosConfig); } - /** - * @deprecated Use htmlApi() instead. This method will be removed in version 2.0.0. - */ - public get = deprecate((config: HtmlApiConfig): AxiosPromise => { - let params: Record = { - ...config.params, - url: config.url, - cookies: config.cookies - }; - - let headers = process_headers(config.headers); - if (Object.keys(config.headers ?? {}).length > 0) { - params.forward_headers = true; - } - - return this.request({ - method: 'GET', - endpoint: HTML_API_URL, - params: process_params(params), - headers: headers, - data: config.data, - retries: config.retries, - timeout: config.timeout, - }); - }, 'ScrapingBeeClient.get() is deprecated. Please use client.htmlApi() instead. This method will be removed in version 2.0.0.'); - - /** - * @deprecated Use htmlApi() instead. This method will be removed in version 2.0.0. - */ - public post = deprecate((config: HtmlApiConfig): AxiosPromise => { - let params: Record = { - ...config.params, - url: config.url, - cookies: config.cookies - }; - - let headers = process_headers(config.headers); - if (Object.keys(config.headers ?? {}).length > 0) { - params.forward_headers = true; - } - - return this.request({ - method: 'POST', - endpoint: HTML_API_URL, - params: process_params(params), - headers: headers, - data: config.data, - retries: config.retries, - timeout: config.timeout, - }); - }, 'ScrapingBeeClient.post() is deprecated. Please use client.htmlApi() instead. This method will be removed in version 2.0.0.'); - public googleSearch(config: GoogleSearchConfig): AxiosPromise { const params: Record = { search: config.search, @@ -456,6 +483,7 @@ export class ScrapingBeeClient { public youtubeMetadata(config: YouTubeMetadataConfig): AxiosPromise { const params: Record = { video_id: config.video_id, + ...config.params, }; return this.request({ @@ -467,7 +495,7 @@ export class ScrapingBeeClient { }); } - public youtubeTranscript(config: YouTubeTranscriptConfig): AxiosPromise { + public youtubeSubtitles(config: YouTubeSubtitlesConfig): AxiosPromise { const params: Record = { video_id: config.video_id, ...config.params, @@ -475,21 +503,52 @@ export class ScrapingBeeClient { return this.request({ method: 'GET', - endpoint: YOUTUBE_TRANSCRIPT_API_URL, + endpoint: YOUTUBE_SUBTITLES_API_URL, params, retries: config.retries, timeout: config.timeout, }); } - public youtubeTrainability(config: YouTubeTrainabilityConfig): AxiosPromise { + public fastSearch(config: FastSearchConfig): AxiosPromise { const params: Record = { - video_id: config.video_id, + search: config.search, + ...config.params, + }; + + return this.request({ + method: 'GET', + endpoint: FAST_SEARCH_API_URL, + params, + retries: config.retries, + timeout: config.timeout, + }); + } + + public amazonPricing(config: AmazonPricingConfig): AxiosPromise { + const params: Record = { + asin: config.asin, + ...config.params, + }; + + return this.request({ + method: 'GET', + endpoint: AMAZON_PRICING_API_URL, + params, + retries: config.retries, + timeout: config.timeout, + }); + } + + public gemini(config: GeminiConfig): AxiosPromise { + const params: Record = { + prompt: config.prompt, + ...config.params, }; return this.request({ method: 'GET', - endpoint: YOUTUBE_TRAINABILITY_API_URL, + endpoint: GEMINI_API_URL, params, retries: config.retries, timeout: config.timeout, diff --git a/src/version.ts b/src/version.ts index f99a7cf..5c35a21 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1 +1 @@ -export const LIB_VERSION = "1.8.3"; +export const LIB_VERSION = "2.0.0"; diff --git a/tests/client.test.ts b/tests/client.test.ts index 45b9ab2..e4a6954 100644 --- a/tests/client.test.ts +++ b/tests/client.test.ts @@ -10,89 +10,6 @@ var mock = new MockAdapter(axios); // HTML API // ============================================ -describe('test_ScrapingBeeClient.get', function () { - var api_key = 'API_KEY'; - var target_url = 'https://httpbin-scrapingbee.cleverapps.io/html'; - var client = new ScrapingBeeClient(api_key); - mock.onGet().reply(200); - - it('should make a simple GET request with correct query params', async function () { - var res = await client.get({ url: target_url }); - assert.deepStrictEqual(res.status, 200); - assert.deepStrictEqual(res.config.params['api_key'], api_key); - assert.deepStrictEqual(res.config.params['url'], target_url); - // @ts-ignore - assert.match(res.config.headers['User-Agent'], /^ScrapingBee-Node\//); - }); - - it('should add the render_js query param', async function () { - var res = await client.get({ url: target_url, params: { render_js: true } }); - assert.deepStrictEqual(res.config.params['render_js'], true); - }); - - it('should prefix header names with Spb- and set forward_headers', async function () { - var res = await client.get({ url: target_url, headers: { 'Content-Type': 'text/html; charset=utf-8' } }); - // @ts-ignore - assert.deepStrictEqual(res.config.headers['Spb-Content-Type'], 'text/html; charset=utf-8'); - // @ts-ignore - assert.deepStrictEqual(res.config.headers['User-Agent'], `ScrapingBee-Node/${LIB_VERSION}`); - assert.deepStrictEqual(res.config.params['forward_headers'], true); - }); - - it('should format the cookies and add them to the query params', async function () { - var cookies = { name1: 'value1', name2: 'value2' }; - var res = await client.get({ url: target_url, cookies: cookies }); - assert.deepStrictEqual(res.config.params['cookies'], 'name1=value1;name2=value2'); - }); - - it('should format the extract_rules and add them to the query params', async function () { - var res = await client.get({ - url: target_url, - params: { - extract_rules: { - title: 'h1', - subtitle: '#subtitle', - }, - }, - }); - assert.deepStrictEqual( - res.config.params['extract_rules'], - '{"title":"h1","subtitle":"#subtitle"}' - ); - }); - - it('should format the js_scenario and add them to the query params', async function () { - var res = await client.get({ - url: target_url, - params: { - js_scenario: { - instructions: [{ click: '#buttonId' }], - }, - }, - }); - assert.deepStrictEqual( - res.config.params['js_scenario'], - '{"instructions":[{"click":"#buttonId"}]}' - ); - }); -}); - -describe('test_ScrapingBeeClient.post', function () { - var api_key = 'API_KEY'; - var target_url = 'https://httpbin-scrapingbee.cleverapps.io/post'; - var client = new ScrapingBeeClient(api_key); - mock.onPost().reply(201); - - it('should make a simple GET request with correct query params', async function () { - var res = await client.post({ url: target_url }); - assert.deepStrictEqual(res.status, 201); - assert.deepStrictEqual(res.config.params['api_key'], api_key); - assert.deepStrictEqual(res.config.params['url'], target_url); - // @ts-ignore - assert.match(res.config.headers['User-Agent'], /^ScrapingBee-Node\//); - }); -}); - describe('test_ScrapingBeeClient.htmlApi', function () { var api_key = 'API_KEY'; var target_url = 'https://httpbin.org/get'; @@ -134,11 +51,21 @@ describe('test_ScrapingBeeClient.htmlApi', function () { it('should handle multiple params correctly', async function () { var res = await client.htmlApi({ url: target_url, - params: { render_js: true, premium_proxy: true, block_ads: true } + params: { render_js: true, premium_proxy: true, block_ads: true, tag: 'my-tag' } }); assert.deepStrictEqual(res.config.params['render_js'], true); assert.deepStrictEqual(res.config.params['premium_proxy'], true); assert.deepStrictEqual(res.config.params['block_ads'], true); + assert.deepStrictEqual(res.config.params['tag'], 'my-tag'); + }); + + it('should handle auto-mode params correctly', async function () { + var res = await client.htmlApi({ + url: target_url, + params: { mode: 'auto', max_cost: 25 } + }); + assert.deepStrictEqual(res.config.params['mode'], 'auto'); + assert.deepStrictEqual(res.config.params['max_cost'], 25); }); it('should prefix header names with Spb- and set forward_headers', async function () { @@ -267,19 +194,40 @@ describe('test_ScrapingBeeClient.googleSearch', function () { search: 'test', params: { add_html: true, + country_code: 'us', + date_range: 'past_week', device: 'mobile', - page: 2, - search_type: 'news', + language: 'en', + latitude: 40.7128, light_request: false, - nfpr: true + longitude: -74.0060, + max_price: 1000, + min_price: 10, + nfpr: true, + page: 2, + pages: 3, + radius: 5000, + search_type: 'shopping', + sort_by: 'price_asc', + tag: 'my-tag' } }); assert.deepStrictEqual(res.config.params['add_html'], true); + assert.deepStrictEqual(res.config.params['country_code'], 'us'); + assert.deepStrictEqual(res.config.params['date_range'], 'past_week'); assert.deepStrictEqual(res.config.params['device'], 'mobile'); + assert.deepStrictEqual(res.config.params['latitude'], 40.7128); + assert.deepStrictEqual(res.config.params['longitude'], -74.006); + assert.deepStrictEqual(res.config.params['max_price'], 1000); + assert.deepStrictEqual(res.config.params['min_price'], 10); assert.deepStrictEqual(res.config.params['page'], 2); - assert.deepStrictEqual(res.config.params['search_type'], 'news'); + assert.deepStrictEqual(res.config.params['pages'], 3); + assert.deepStrictEqual(res.config.params['radius'], 5000); + assert.deepStrictEqual(res.config.params['search_type'], 'shopping'); + assert.deepStrictEqual(res.config.params['sort_by'], 'price_asc'); assert.deepStrictEqual(res.config.params['light_request'], false); assert.deepStrictEqual(res.config.params['nfpr'], true); + assert.deepStrictEqual(res.config.params['tag'], 'my-tag'); }); }); @@ -314,19 +262,23 @@ describe('test_ScrapingBeeClient.amazonSearch', function () { query: 'laptop', params: { add_html: true, + autoselect_variant: true, country: 'us', currency: 'USD', device: 'desktop', pages: 2, sort_by: 'price_low_to_high', - start_page: 1 + start_page: 1, + tag: 'my-tag' } }); assert.deepStrictEqual(res.config.params['add_html'], true); + assert.deepStrictEqual(res.config.params['autoselect_variant'], true); assert.deepStrictEqual(res.config.params['country'], 'us'); assert.deepStrictEqual(res.config.params['currency'], 'USD'); assert.deepStrictEqual(res.config.params['pages'], 2); assert.deepStrictEqual(res.config.params['sort_by'], 'price_low_to_high'); + assert.deepStrictEqual(res.config.params['tag'], 'my-tag'); }); }); @@ -366,12 +318,14 @@ describe('test_ScrapingBeeClient.amazonProduct', function () { device: 'mobile', language: 'en', light_request: false, - screenshot: true + screenshot: true, + tag: 'my-tag' } }); assert.deepStrictEqual(res.config.params['autoselect_variant'], true); assert.deepStrictEqual(res.config.params['screenshot'], true); assert.deepStrictEqual(res.config.params['light_request'], false); + assert.deepStrictEqual(res.config.params['tag'], 'my-tag'); }); }); @@ -411,13 +365,17 @@ describe('test_ScrapingBeeClient.walmartSearch', function () { max_price: 1000, min_price: 100, screenshot: true, - store_id: '12345' + start_page: 2, + store_id: '12345', + tag: 'my-tag' } }); assert.deepStrictEqual(res.config.params['delivery_zip'], '10001'); assert.deepStrictEqual(res.config.params['max_price'], 1000); assert.deepStrictEqual(res.config.params['min_price'], 100); assert.deepStrictEqual(res.config.params['fulfillment_speed'], 'today'); + assert.deepStrictEqual(res.config.params['start_page'], 2); + assert.deepStrictEqual(res.config.params['tag'], 'my-tag'); }); }); @@ -456,12 +414,14 @@ describe('test_ScrapingBeeClient.walmartProduct', function () { domain: 'com', light_request: false, screenshot: true, - store_id: '12345' + store_id: '12345', + tag: 'my-tag' } }); assert.deepStrictEqual(res.config.params['delivery_zip'], '10001'); assert.deepStrictEqual(res.config.params['device'], 'tablet'); assert.deepStrictEqual(res.config.params['screenshot'], true); + assert.deepStrictEqual(res.config.params['tag'], 'my-tag'); }); }); @@ -496,12 +456,14 @@ describe('test_ScrapingBeeClient.chatGPT', function () { params: { add_html: true, country_code: 'us', - search: true + search: true, + tag: 'my-tag' } }); assert.deepStrictEqual(res.config.params['add_html'], true); assert.deepStrictEqual(res.config.params['country_code'], 'us'); assert.deepStrictEqual(res.config.params['search'], true); + assert.deepStrictEqual(res.config.params['tag'], 'my-tag'); }); }); @@ -541,7 +503,8 @@ describe('test_ScrapingBeeClient.youtubeSearch', function () { sort_by: 'view_count', type: 'video', subtitles: true, - live: false + live: false, + tag: 'my-tag' } }); assert.deepStrictEqual(res.config.params['4k'], true); @@ -549,6 +512,7 @@ describe('test_ScrapingBeeClient.youtubeSearch', function () { assert.deepStrictEqual(res.config.params['duration'], '4-20'); assert.deepStrictEqual(res.config.params['upload_date'], 'this_week'); assert.deepStrictEqual(res.config.params['subtitles'], true); + assert.deepStrictEqual(res.config.params['tag'], 'my-tag'); }); }); @@ -569,19 +533,28 @@ describe('test_ScrapingBeeClient.youtubeMetadata', function () { assert.deepStrictEqual(res.config.params['api_key'], api_key); assert.deepStrictEqual(res.config.params['video_id'], 'dQw4w9WgXcQ'); }); + + it('should forward optional params', async function () { + var res = await client.youtubeMetadata({ + video_id: 'dQw4w9WgXcQ', + params: { tag: 'my-tag' } + }); + assert.deepStrictEqual(res.config.params['video_id'], 'dQw4w9WgXcQ'); + assert.deepStrictEqual(res.config.params['tag'], 'my-tag'); + }); }); // ============================================ -// YouTube Transcript API +// YouTube Subtitles API // ============================================ -describe('test_ScrapingBeeClient.youtubeTranscript', function () { +describe('test_ScrapingBeeClient.youtubeSubtitles', function () { var api_key = 'API_KEY'; var client = new ScrapingBeeClient(api_key); mock.onGet().reply(200); it('should make a request with correct params', async function () { - var res = await client.youtubeTranscript({ + var res = await client.youtubeSubtitles({ video_id: 'dQw4w9WgXcQ', params: { language: 'en' } }); @@ -592,39 +565,155 @@ describe('test_ScrapingBeeClient.youtubeTranscript', function () { }); it('should work with only required param', async function () { - var res = await client.youtubeTranscript({ video_id: 'dQw4w9WgXcQ' }); + var res = await client.youtubeSubtitles({ video_id: 'dQw4w9WgXcQ' }); assert.deepStrictEqual(res.config.params['video_id'], 'dQw4w9WgXcQ'); }); it('should handle all optional params', async function () { - var res = await client.youtubeTranscript({ + var res = await client.youtubeSubtitles({ video_id: 'dQw4w9WgXcQ', params: { language: 'es', - transcript_origin: 'uploader_provided' + subtitle_origin: 'uploader_provided', + tag: 'my-tag' } }); assert.deepStrictEqual(res.config.params['language'], 'es'); - assert.deepStrictEqual(res.config.params['transcript_origin'], 'uploader_provided'); + assert.deepStrictEqual(res.config.params['subtitle_origin'], 'uploader_provided'); + assert.deepStrictEqual(res.config.params['tag'], 'my-tag'); }); }); // ============================================ -// YouTube Trainability API +// Fast Search API // ============================================ -describe('test_ScrapingBeeClient.youtubeTrainability', function () { +describe('test_ScrapingBeeClient.fastSearch', function () { var api_key = 'API_KEY'; var client = new ScrapingBeeClient(api_key); mock.onGet().reply(200); it('should make a request with correct params', async function () { - var res = await client.youtubeTrainability({ - video_id: 'dQw4w9WgXcQ' + var res = await client.fastSearch({ + search: 'test query', + params: { language: 'en', country_code: 'us' } }); assert.deepStrictEqual(res.status, 200); assert.deepStrictEqual(res.config.params['api_key'], api_key); - assert.deepStrictEqual(res.config.params['video_id'], 'dQw4w9WgXcQ'); + assert.deepStrictEqual(res.config.params['search'], 'test query'); + assert.deepStrictEqual(res.config.params['language'], 'en'); + assert.deepStrictEqual(res.config.params['country_code'], 'us'); + }); + + it('should work with only required param', async function () { + var res = await client.fastSearch({ search: 'test' }); + assert.deepStrictEqual(res.config.params['search'], 'test'); + }); + + it('should handle all optional params', async function () { + var res = await client.fastSearch({ + search: 'test', + params: { + page: 2, + country_code: 'fr', + language: 'fr', + tag: 'my-tag' + } + }); + assert.deepStrictEqual(res.config.params['page'], 2); + assert.deepStrictEqual(res.config.params['country_code'], 'fr'); + assert.deepStrictEqual(res.config.params['language'], 'fr'); + assert.deepStrictEqual(res.config.params['tag'], 'my-tag'); + }); +}); + +// ============================================ +// Amazon Pricing API +// ============================================ + +describe('test_ScrapingBeeClient.amazonPricing', function () { + var api_key = 'API_KEY'; + var client = new ScrapingBeeClient(api_key); + mock.onGet().reply(200); + + it('should make a request with correct params', async function () { + var res = await client.amazonPricing({ + asin: 'B0D2Q9397Y', + params: { domain: 'com' } + }); + assert.deepStrictEqual(res.status, 200); + assert.deepStrictEqual(res.config.params['api_key'], api_key); + assert.deepStrictEqual(res.config.params['asin'], 'B0D2Q9397Y'); + assert.deepStrictEqual(res.config.params['domain'], 'com'); + }); + + it('should work with only required param', async function () { + var res = await client.amazonPricing({ asin: 'B0D2Q9397Y' }); + assert.deepStrictEqual(res.config.params['asin'], 'B0D2Q9397Y'); + }); + + it('should handle all optional params', async function () { + var res = await client.amazonPricing({ + asin: 'B0D2Q9397Y', + params: { + add_html: true, + country: 'us', + currency: 'USD', + device: 'desktop', + domain: 'com', + language: 'en', + light_request: false, + tag: 'my-tag', + zip_code: '10001' + } + }); + assert.deepStrictEqual(res.config.params['add_html'], true); + assert.deepStrictEqual(res.config.params['country'], 'us'); + assert.deepStrictEqual(res.config.params['currency'], 'USD'); + assert.deepStrictEqual(res.config.params['device'], 'desktop'); + assert.deepStrictEqual(res.config.params['light_request'], false); + assert.deepStrictEqual(res.config.params['zip_code'], '10001'); + assert.deepStrictEqual(res.config.params['tag'], 'my-tag'); + }); +}); + +// ============================================ +// Gemini API +// ============================================ + +describe('test_ScrapingBeeClient.gemini', function () { + var api_key = 'API_KEY'; + var client = new ScrapingBeeClient(api_key); + mock.onGet().reply(200); + + it('should make a request with correct params', async function () { + var res = await client.gemini({ + prompt: 'What is web scraping?', + params: { country_code: 'us' } + }); + assert.deepStrictEqual(res.status, 200); + assert.deepStrictEqual(res.config.params['api_key'], api_key); + assert.deepStrictEqual(res.config.params['prompt'], 'What is web scraping?'); + assert.deepStrictEqual(res.config.params['country_code'], 'us'); + }); + + it('should work with only required param', async function () { + var res = await client.gemini({ prompt: 'Hello' }); + assert.deepStrictEqual(res.config.params['prompt'], 'Hello'); + }); + + it('should handle all optional params', async function () { + var res = await client.gemini({ + prompt: 'Explain AI', + params: { + add_html: true, + country_code: 'us', + tag: 'my-tag' + } + }); + assert.deepStrictEqual(res.config.params['add_html'], true); + assert.deepStrictEqual(res.config.params['country_code'], 'us'); + assert.deepStrictEqual(res.config.params['tag'], 'my-tag'); }); });