add dist

webrecorder · Nov 20, 2024 · 1740f61 · 1740f61
1 parent adc2618
commit 1740f61
Show file tree

Hide file tree

Showing 85 changed files with 1,684 additions and 0 deletions.
diff --git a/dist/index.js b/dist/index.js
diff --git a/dist/sw.js b/dist/sw.js
diff --git a/dist/swlib.js b/dist/swlib.js
diff --git a/dist/types/adblockcss.d.ts b/dist/types/adblockcss.d.ts
@@ -0,0 +1,12 @@
+export declare function getAdBlockCSSResponse(fullDomain: string, adblockUrl: string): Promise<Response>;
+export declare class ByLineTransform {
+    _buffer: string[];
+    _lastChunkEndedWithCR: boolean;
+    decoder: TextDecoder;
+    transform(chunkArray: Uint8Array, controller: TransformStreamDefaultController): void;
+    flush(controller: TransformStreamDefaultController): void;
+}
+export declare class ByLineStream extends TransformStream<Uint8Array, string> {
+    constructor();
+}
+//# sourceMappingURL=adblockcss.d.ts.map
diff --git a/dist/types/adblockcss.d.ts.map b/dist/types/adblockcss.d.ts.map
diff --git a/dist/types/api.d.ts b/dist/types/api.d.ts
@@ -0,0 +1,24 @@
+import { Path } from "path-parser";
+import { type SWCollections } from "./swmain";
+type RouteMatch = Record<string, any>;
+declare class APIRouter {
+    routes: Record<string, Record<string, Path>>;
+    constructor(paths: Record<string, string | [string, string]>);
+    match(url: string, method?: string): RouteMatch | {
+        _route: null;
+    };
+}
+declare class API {
+    router: APIRouter;
+    collections: SWCollections;
+    constructor(collections: SWCollections);
+    get routes(): Record<string, string | [string, string]>;
+    apiResponse(url: string, request: Request, event: FetchEvent): Promise<Response>;
+    handleApi(request: Request, params: RouteMatch, event: FetchEvent): Promise<any>;
+    listAll(filter?: string | null): Promise<{
+        colls: any[];
+    }>;
+    makeResponse(response: Response, status?: number): Response;
+}
+export { API };
+//# sourceMappingURL=api.d.ts.map
diff --git a/dist/types/api.d.ts.map b/dist/types/api.d.ts.map
diff --git a/dist/types/archivedb.d.ts b/dist/types/archivedb.d.ts
@@ -0,0 +1,162 @@
+import { type IDBPDatabase, type IDBPTransaction } from "idb/with-async-ittr";
+import { ArchiveResponse } from "./response";
+import { type DBStore, type DigestRefCount, type PageEntry, type ResAPIResponse, type ResourceEntry } from "./types";
+import { type ArchiveRequest } from "./request";
+import { type BaseAsyncIterReader } from "warcio";
+export type ADBOpts = {
+    minDedupSize?: number | undefined;
+    noRefCounts?: unknown;
+    noFuzzyCheck?: boolean;
+    noRevisits?: boolean;
+    pageId?: string;
+};
+export type ADBType = {
+    pages: {
+        key: string;
+        value: PageEntry & {
+            size?: number;
+        };
+        indexes: {
+            url: string;
+            ts: string;
+            state: number;
+        };
+    };
+    pageLists: {
+        key: string;
+        value: {
+            pages?: unknown[];
+            show?: boolean;
+            title?: string | undefined;
+            desc?: string | undefined;
+            slug?: string | undefined;
+        };
+    };
+    curatedPages: {
+        key: string;
+        value: PageEntry;
+        indexes: {
+            listPages: [string, string];
+        };
+    };
+    resources: {
+        key: [string, string];
+        value: ResourceEntry;
+        indexes: {
+            pageId: string;
+            mimeStatusUrl: [string, string, string];
+        };
+    };
+    payload: {
+        key: string;
+        value: {
+            digest: string;
+            payload: Uint8Array | null;
+        };
+        indexes: {
+            digest: string;
+        };
+    };
+    digestRef: {
+        key: string;
+        value: DigestRefCount | null;
+        indexes: {
+            digest: string;
+        };
+    };
+};
+export declare class ArchiveDB implements DBStore {
+    name: string;
+    minDedupSize: number;
+    version: number;
+    autoHttpsCheck: boolean;
+    useRefCounts: boolean;
+    allowRepeats: boolean;
+    repeatTracker: RepeatTracker | null;
+    fuzzyPrefixSearch: boolean;
+    initing: Promise<void>;
+    db: IDBPDatabase<ADBType> | null;
+    constructor(name: string, opts?: ADBOpts | undefined);
+    init(): Promise<void>;
+    _initDB(db: IDBPDatabase<ADBType>, oldV: number, _newV: number | null, _tx?: IDBPTransaction<ADBType, (keyof ADBType)[], "readwrite" | "versionchange">): void;
+    clearAll(): Promise<void>;
+    close(): void;
+    delete(): Promise<void>;
+    addPage(page: PageEntry, tx?: IDBPTransaction<ADBType, [keyof ADBType], "readwrite"> | null): Promise<string>;
+    addPages(pages: PageEntry[], pagesTable?: keyof ADBType, update?: boolean): Promise<void>;
+    createPageList(data: {
+        title?: string;
+        desc?: string;
+        description?: string;
+        id?: string;
+        slug?: string;
+    }): Promise<string>;
+    addCuratedPageList(listInfo: Record<string, unknown>, pages: PageEntry[]): Promise<void>;
+    addCuratedPageLists(pageLists: {
+        [k: string]: PageEntry[] | undefined;
+    }[], pageKey?: string, filter?: string): Promise<void>;
+    convertCuratedPagesToV2(db: IDBPDatabase<ADBType & {
+        pages: {
+            key: string;
+            value: {
+                page?: PageEntry;
+            } & PageEntry;
+        };
+        curatedPages: {
+            key: string;
+            value: {
+                page?: PageEntry;
+            } & PageEntry;
+        };
+    }>): Promise<void>;
+    getCuratedPagesByList(): Promise<{
+        pages?: unknown[];
+        show?: boolean;
+        title?: string | undefined;
+        desc?: string | undefined;
+        slug?: string | undefined;
+    }[]>;
+    newPageId(): string;
+    getAllPages(): Promise<(PageEntry & {
+        size?: number;
+    })[]>;
+    getPages(pages: string[]): Promise<PageEntry[]>;
+    getTimestampsByURL(url: string): Promise<string[]>;
+    getPagesWithState(state: number): Promise<(PageEntry & {
+        size?: number;
+    })[]>;
+    getVerifyInfo(): Promise<{}>;
+    addVerifyData(_prefix: string | undefined, _id: string, _expected: string, _actual?: string | null, _log?: boolean): Promise<void>;
+    addVerifyDataList(_prefix: string, _datalist: unknown[]): Promise<void>;
+    dedupResource(digest: string, payload: Uint8Array | null | undefined, tx: IDBPTransaction<ADBType, (keyof ADBType)[], "readwrite">, count?: number): Promise<DigestRefCount | null>;
+    addResources(datas: ResourceEntry[]): Promise<void>;
+    getFuzzyUrl(result: ResourceEntry): ResourceEntry | null;
+    addResource(data: ResourceEntry): Promise<boolean>;
+    getResource(request: ArchiveRequest, _prefix: string, event: FetchEvent, opts?: ADBOpts): Promise<ArchiveResponse | Response | null>;
+    loadPayload(result: ResourceEntry, _opts: ADBOpts): Promise<BaseAsyncIterReader | Uint8Array | null>;
+    isSelfRedirect(url: string, result: ResourceEntry | undefined): boolean;
+    lookupUrl(url: string, ts?: number, opts?: ADBOpts): Promise<ResourceEntry | null>;
+    lookupQueryPrefix(url: string, opts: ADBOpts): Promise<ResourceEntry | null>;
+    resJson(res: ResourceEntry): ResAPIResponse;
+    resourcesByPage(pageId: string): Promise<ResourceEntry[]>;
+    resourcesByPages2(pageIds: string[]): AsyncGenerator<ResourceEntry, void, unknown>;
+    resourcesByPages(pageIds: string[]): AsyncGenerator<ResourceEntry, void, unknown>;
+    matchAny<S extends keyof ADBType>(storeName: S, indexName: ADBType[S] extends {
+        indexes: {};
+    } ? keyof ADBType[S]["indexes"] | null : null, sortedKeys: string[], subKey?: number, openBound?: boolean): AsyncGenerator<ADBType[S]["value"], void, unknown>;
+    resourcesByUrlAndMime(url: string, mimes: string, count?: number, prefix?: boolean, fromUrl?: string, fromTs?: string): Promise<ResAPIResponse[]>;
+    resourcesByMime(mimesStr: string, count?: number, fromMime?: string, fromUrl?: string, fromStatus?: number): Promise<ResAPIResponse[]>;
+    deletePage(id: string): Promise<{
+        pageSize: number;
+        dedupSize: number;
+    }>;
+    deletePageResources(pageId: string): Promise<number>;
+    prefixUpperBound(url: string): string;
+    getLookupRange(url: string, type: string, fromUrl?: string, fromTs?: string): IDBKeyRange;
+}
+declare class RepeatTracker {
+    repeats: Record<string, Record<string, number>>;
+    getSkipCount(event: FetchEvent, url: string, method: string): number;
+}
+export {};
+//# sourceMappingURL=archivedb.d.ts.map
diff --git a/dist/types/archivedb.d.ts.map b/dist/types/archivedb.d.ts.map
diff --git a/dist/types/baseparser.d.ts b/dist/types/baseparser.d.ts
@@ -0,0 +1,41 @@
+import { type CollMetadata, type ArchiveLoader, type DBStore, type PageEntry } from "./types";
+export type ResourceEntry = {
+    url: string;
+    ts: number;
+    digest?: string | null;
+    status?: number;
+    mime?: string;
+    respHeaders?: Record<string, string> | null;
+    reqHeaders?: Record<string, string> | null;
+    recordDigest?: string | null;
+    payload?: Uint8Array | null;
+    reader?: AsyncIterable<Uint8Array> | Iterable<Uint8Array> | null;
+    referrer?: string | null;
+    extraOpts?: Record<string, any> | null;
+    pageId?: string | null;
+    origURL?: string | null;
+    origTS?: number | null;
+    source?: object;
+    requestUrl?: string | null;
+    method?: string | null;
+    requestBody?: Uint8Array;
+    loaded?: boolean;
+};
+declare abstract class BaseParser implements ArchiveLoader {
+    batchSize: number;
+    promises: Promise<void>[];
+    batch: ResourceEntry[];
+    count: number;
+    dupeSet: Set<string>;
+    db: any;
+    constructor(batchSize?: number);
+    addPage(page: PageEntry): void;
+    isBatchFull(): boolean;
+    addResource(res: ResourceEntry): void;
+    flush(): void;
+    finishIndexing(): Promise<void>;
+    _finishLoad(): void;
+    abstract load(db: DBStore, progressUpdateCallback?: any, totalLength?: number): Promise<CollMetadata | undefined>;
+}
+export { BaseParser };
+//# sourceMappingURL=baseparser.d.ts.map
diff --git a/dist/types/baseparser.d.ts.map b/dist/types/baseparser.d.ts.map
diff --git a/dist/types/blockloaders.d.ts b/dist/types/blockloaders.d.ts
@@ -0,0 +1,30 @@
+export type ResponseAbort = {
+    response: Response;
+    abort: AbortController | null;
+};
+export type BlockLoaderExtra = {
+    arrayBuffer?: Uint8Array;
+    publicUrl?: string;
+    fileHandle?: FileSystemFileHandle;
+};
+export type BlockLoaderOpts = {
+    url: string;
+    headers?: Record<string, string> | Headers;
+    extra?: BlockLoaderExtra;
+    size?: number;
+    blob?: Blob;
+};
+export declare function createLoader(opts: BlockLoaderOpts): Promise<BaseLoader>;
+export declare abstract class BaseLoader {
+    canLoadOnDemand: boolean;
+    headers: Record<string, string> | Headers;
+    length: number | null;
+    constructor(canLoadOnDemand: boolean);
+    abstract doInitialFetch(tryHead: boolean, skipRange: boolean): Promise<ResponseAbort>;
+    abstract getLength(): Promise<number>;
+    abstract getRange(offset: number, length: number, streaming: boolean, signal?: AbortSignal | null): Promise<Uint8Array | ReadableStream<Uint8Array>>;
+    abstract get isValid(): boolean;
+}
+export declare function getReadableStreamFromIter(stream: AsyncIterable<Uint8Array>): ReadableStream<any>;
+export declare function getReadableStreamFromArray(array: Uint8Array): ReadableStream<any>;
+//# sourceMappingURL=blockloaders.d.ts.map