LocalLm types documentation
    Preparing search index...

    Interface LmProvider

    Defines the structure and behavior of an LM Provider.

    LmProvider

    Identifier for the LM provider.

    API utility being used.

    The URL endpoint for the provider's server.

    The key used for authentication with the provider's API.

    Active model configuration.

    List of available model configurations.

    Retrieves information about available server config.

    Retrieves information about available models.

    Loads a model by name, with optional context.

    Unload a model

    Makes an inference based on provided prompt and parameters.

    Aborts a currently running inference task.

    Callback when a new token is received

    Callback triggered when inference starts.

    Callback triggered when inference ends.

    Callback triggered on errors during inference.

    const lmProvider: LmProvider = {
      name: 'koboldcpp',
      api: useApi(),
      serverUrl: 'http://example.com/api',
      apiKey: 'your-api-key',
      model: { name: 'gpt-3', ctx: 2048 },
      models: [{ name: 'gpt-3', ctx: 2048 }],
      info: async () => ({ config: 'some-config' }),
      modelsInfo: async () => {},
      loadModel: async (name, ctx, urls, onLoadProgress) => {},
      infer: async (prompt, params, options) => ({ text: 'result', stats: {}, serverStats: {} }),
      abort: async () => {},
      onToken: (t) => console.log(t),
      onStartEmit: (data) => console.log(data),
      onEndEmit: (result) => console.log(result),
      onError: (err) => console.error(err)
    };
    interface LmProvider {
        abort: () => Promise<void>;
        api: {
            addHeader: (key: string, val: string) => void;
            csrfToken: () => string | null;
            del: <T>(uri: string, verbose?: boolean) => Promise<ApiResponse<T>>;
            get: <T>(uri: string, verbose?: boolean) => Promise<ApiResponse<T>>;
            hasCsrfCookie: () => boolean;
            onResponse: (hook: OnResponseHook) => void;
            patch: <T>(
                uri: string,
                payload: any[] | Record<string, any>,
                verbose?: boolean,
            ) => Promise<ApiResponse<T>>;
            post: <T>(
                uri: string,
                payload: any[] | FormData | Record<string, any>,
                multipart?: boolean,
                verbose?: boolean,
            ) => Promise<ApiResponse<T>>;
            postSse: <T>(
                uri: string,
                payload: any[] | FormData | Record<string, any>,
                onChunk: (payload: T) => void,
                abortController: AbortController,
                parseJson?: boolean,
                multipart?: boolean,
                verbose?: boolean,
                debug?: boolean,
            ) => Promise<void>;
            put: <T>(
                uri: string,
                payload: any[] | Record<string, any>,
                verbose?: boolean,
            ) => Promise<ApiResponse<T>>;
            removeHeader: (key: string) => void;
            setCsrfToken: (token: string) => void;
            setCsrfTokenFromCookie: (verbose?: boolean) => boolean;
        };
        apiKey: string;
        defaults?: LmDefaults;
        infer: (
            prompt: string,
            params: InferenceParams,
            options?: InferenceOptions,
        ) => Promise<InferenceResult>;
        loadModel: (
            name: string,
            ctx?: number,
            urls?: string | string[],
            onLoadProgress?: OnLoadProgress,
        ) => Promise<void>;
        model: ModelConf;
        modelInfo: () => Promise<ModelConf<Record<string, any>>>;
        models: ModelConf<Record<string, any>>[];
        modelsInfo: () => Promise<ModelConf<Record<string, any>>[]>;
        name: string;
        onEndEmit?: (result: InferenceResult) => void;
        onError?: (err: string) => void;
        onStartEmit?: (data: IngestionStats) => void;
        onToken?: (t: string) => void;
        serverUrl: string;
        unloadModel: (name: string) => Promise<void>;
    }
    Index

    Properties

    abort api apiKey defaults? infer loadModel model modelInfo models modelsInfo name onEndEmit? onError? onStartEmit? onToken? serverUrl unloadModel

    Properties

    abort: () => Promise<void>
    api: {
        addHeader: (key: string, val: string) => void;
        csrfToken: () => string | null;
        del: <T>(uri: string, verbose?: boolean) => Promise<ApiResponse<T>>;
        get: <T>(uri: string, verbose?: boolean) => Promise<ApiResponse<T>>;
        hasCsrfCookie: () => boolean;
        onResponse: (hook: OnResponseHook) => void;
        patch: <T>(
            uri: string,
            payload: any[] | Record<string, any>,
            verbose?: boolean,
        ) => Promise<ApiResponse<T>>;
        post: <T>(
            uri: string,
            payload: any[] | FormData | Record<string, any>,
            multipart?: boolean,
            verbose?: boolean,
        ) => Promise<ApiResponse<T>>;
        postSse: <T>(
            uri: string,
            payload: any[] | FormData | Record<string, any>,
            onChunk: (payload: T) => void,
            abortController: AbortController,
            parseJson?: boolean,
            multipart?: boolean,
            verbose?: boolean,
            debug?: boolean,
        ) => Promise<void>;
        put: <T>(
            uri: string,
            payload: any[] | Record<string, any>,
            verbose?: boolean,
        ) => Promise<ApiResponse<T>>;
        removeHeader: (key: string) => void;
        setCsrfToken: (token: string) => void;
        setCsrfTokenFromCookie: (verbose?: boolean) => boolean;
    }
    apiKey: string
    defaults?: LmDefaults
    infer: (
        prompt: string,
        params: InferenceParams,
        options?: InferenceOptions,
    ) => Promise<InferenceResult>

    Makes an inference based on provided prompt and parameters.

    Type Declaration

    loadModel: (
        name: string,
        ctx?: number,
        urls?: string | string[],
        onLoadProgress?: OnLoadProgress,
    ) => Promise<void>
    model: ModelConf
    modelInfo: () => Promise<ModelConf<Record<string, any>>>
    models: ModelConf<Record<string, any>>[]
    modelsInfo: () => Promise<ModelConf<Record<string, any>>[]>
    name: string
    onEndEmit?: (result: InferenceResult) => void
    onError?: (err: string) => void
    onStartEmit?: (data: IngestionStats) => void
    onToken?: (t: string) => void
    serverUrl: string
    unloadModel: (name: string) => Promise<void>

    Settings

    Member Visibility

    On This Page

    Properties