/* auto-generated by NAPI-RS */
/* eslint-disable */
export declare class AddedToken {
  constructor(token: string, isSpecial: boolean, options?: AddedTokenOptions | undefined | null)
  getContent(): string
}

export declare class BPE {
  static empty(): Model
  static init(vocab: Record<string, number>, merges: Merges, options?: BpeOptions | undefined | null): Model
  static fromFile(vocab: string, merges: string, options?: BpeOptions | undefined | null): Promise<Model>
}
export type Bpe = BPE

/** Decoder */
export declare class Decoder {
  decode(tokens: Array<string>): string
}

export declare class Encoding {
  constructor()
  getLength(): number
  getNSequences(): number
  getIds(): Array<number>
  getTypeIds(): Array<number>
  getAttentionMask(): Array<number>
  getSpecialTokensMask(): Array<number>
  getTokens(): Array<string>
  getOffsets(): Array<Array<number>>
  getWordIds(): Array<number | undefined | null>
  charToToken(pos: number, seqId?: number | undefined | null): number | null
  charToWord(pos: number, seqId?: number | undefined | null): number | null
  pad(length: number, options?: PaddingOptions | undefined | null): void
  truncate(
    length: number,
    stride?: number | undefined | null,
    direction?: string | TruncationDirection | undefined | null,
  ): void
  wordToTokens(word: number, seqId?: number | undefined | null): [number, number] | null | undefined
  wordToChars(word: number, seqId?: number | undefined | null): [number, number] | null | undefined
  tokenToChars(token: number): [number, number] | null | undefined
  tokenToWord(token: number): number | null
  getOverflowing(): Array<Encoding>
  getSequenceIds(): Array<number | undefined | null>
  tokenToSequence(token: number): number | null
}
export type JsEncoding = Encoding

export declare class Model {}

/** Normalizer */
export declare class Normalizer {
  normalizeString(sequence: string): string
}

/** PreTokenizers */
export declare class PreTokenizer {
  preTokenizeString(sequence: string): [string, [number, number]][]
}

export declare class Processor {}

export declare class Tokenizer {
  constructor(model: Model)
  setPreTokenizer(preTokenizer: PreTokenizer): void
  setDecoder(decoder: Decoder): void
  setModel(model: Model): void
  setPostProcessor(postProcessor: Processor): void
  setNormalizer(normalizer: Normalizer): void
  save(path: string, pretty?: boolean | undefined | null): void
  addAddedTokens(tokens: Array<AddedToken>): number
  addTokens(tokens: Array<string>): number
  encode(
    sentence: InputSequence,
    pair?: InputSequence | null,
    encodeOptions?: EncodeOptions | undefined | null,
  ): Promise<JsEncoding>
  encodeBatch(sentences: EncodeInput[], encodeOptions?: EncodeOptions | undefined | null): Promise<JsEncoding[]>
  decode(ids: Array<number>, skipSpecialTokens: boolean): Promise<string>
  decodeBatch(ids: Array<Array<number>>, skipSpecialTokens: boolean): Promise<string[]>
  static fromString(s: string): Tokenizer
  static fromFile(file: string): Tokenizer
  addSpecialTokens(tokens: Array<string>): void
  setTruncation(maxLength: number, options?: TruncationOptions | undefined | null): void
  disableTruncation(): void
  setPadding(options?: PaddingOptions | undefined | null): void
  disablePadding(): void
  getDecoder(): Decoder | null
  getNormalizer(): Normalizer | null
  getPreTokenizer(): PreTokenizer | null
  getPostProcessor(): Processor | null
  getVocab(withAddedTokens?: boolean | undefined | null): Record<string, number>
  getVocabSize(withAddedTokens?: boolean | undefined | null): number
  idToToken(id: number): string | null
  tokenToId(token: string): number | null
  train(files: Array<string>): void
  runningTasks(): number
  postProcess(
    encoding: Encoding,
    pair?: Encoding | undefined | null,
    addSpecialTokens?: boolean | undefined | null,
  ): Encoding
}

export declare class Trainer {}

export declare class Unigram {
  static init(vocab: Array<[string, number]>, options?: UnigramOptions | undefined | null): Model
  static empty(): Model
}

export declare class WordLevel {
  static init(vocab: Record<string, number>, options?: WordLevelOptions | undefined | null): Model
  static empty(): WordLevel
  static fromFile(vocab: string, options?: WordLevelOptions | undefined | null): Promise<Model>
}

export declare class WordPiece {
  static init(vocab: Record<string, number>, options?: WordPieceOptions | undefined | null): Model
  static empty(): WordPiece
  static fromFile(vocab: string, options?: WordPieceOptions | undefined | null): Promise<Model>
}

export interface AddedTokenOptions {
  singleWord?: boolean
  leftStrip?: boolean
  rightStrip?: boolean
  normalized?: boolean
}

/**
 * bert_normalizer(options?: {
 *   cleanText?: bool = true,
 *   handleChineseChars?: bool = true,
 *   stripAccents?: bool = true,
 *   lowercase?: bool = true
 * })
 */
export declare function bertNormalizer(options?: BertNormalizerOptions | undefined | null): Normalizer

export interface BertNormalizerOptions {
  cleanText?: boolean
  handleChineseChars?: boolean
  stripAccents?: boolean
  lowercase?: boolean
}

export declare function bertPreTokenizer(): PreTokenizer

export declare function bertProcessing(sep: [string, number], cls: [string, number]): Processor

export declare function bpeDecoder(suffix?: string | undefined | null): Decoder

export interface BpeOptions {
  cacheCapacity?: number
  dropout?: number
  unkToken?: string
  continuingSubwordPrefix?: string
  endOfWordSuffix?: string
  fuseUnk?: boolean
  byteFallback?: boolean
}

export declare function byteFallbackDecoder(): Decoder

export declare function byteLevelAlphabet(): Array<string>

/** byte_level(addPrefixSpace: bool = true, useRegex: bool = true) */
export declare function byteLevelPreTokenizer(
  addPrefixSpace?: boolean | undefined | null,
  useRegex?: boolean | undefined | null,
): PreTokenizer

export declare function byteLevelProcessing(trimOffsets?: boolean | undefined | null): Processor

export declare function charDelimiterSplit(delimiter: string): PreTokenizer

export declare function ctcDecoder(
  padToken?: string = '<pad>',
  wordDelimiterToken?: string | undefined | null,
  cleanup?: boolean | undefined | null,
): Decoder

export declare function digitsPreTokenizer(individualDigits?: boolean | undefined | null): PreTokenizer

export interface EncodeOptions {
  isPretokenized?: boolean
  addSpecialTokens?: boolean
}

export declare function fuseDecoder(): Decoder

export interface JsFromPretrainedParameters {
  revision?: string
  authToken?: string
}

export declare const enum JsSplitDelimiterBehavior {
  Removed = 'Removed',
  Isolated = 'Isolated',
  MergedWithPrevious = 'MergedWithPrevious',
  MergedWithNext = 'MergedWithNext',
  Contiguous = 'Contiguous',
}

export declare function lowercase(): Normalizer

export declare function mergeEncodings(
  encodings: Array<Encoding>,
  growingOffsets?: boolean | undefined | null,
): Encoding

export declare function metaspaceDecoder(
  replacement?: string = '▁',
  prependScheme?: prepend_scheme = 'always',
  split?: split = true,
): Decoder

export declare function metaspacePreTokenizer(
  replacement?: string = '▁',
  prependScheme?: prepend_scheme = 'always',
  split?: split = true,
): PreTokenizer

export declare function nfcNormalizer(): Normalizer

export declare function nfdNormalizer(): Normalizer

export declare function nfkcNormalizer(): Normalizer

export declare function nfkdNormalizer(): Normalizer

export declare function nmt(): Normalizer

export declare const enum PaddingDirection {
  Left = 0,
  Right = 1,
}

export interface PaddingOptions {
  maxLength?: number
  direction?: string | PaddingDirection
  padToMultipleOf?: number
  padId?: number
  padTypeId?: number
  padToken?: string
}

export declare function precompiled(bytes: Array<number>): Normalizer

export declare function prependNormalizer(prepend: string): Normalizer

export declare function punctuationPreTokenizer(behavior?: string | undefined | null): PreTokenizer

export declare function replace(pattern: string, content: string): Normalizer

export declare function replaceDecoder(pattern: string, content: string): Decoder

export declare function robertaProcessing(
  sep: [string, number],
  cls: [string, number],
  trimOffsets?: boolean | undefined | null,
  addPrefixSpace?: boolean | undefined | null,
): Processor

export declare function sequenceDecoder(decoders: Array<Decoder>): Decoder

export declare function sequenceNormalizer(normalizers: Array<Normalizer>): Normalizer

export declare function sequencePreTokenizer(preTokenizers: Array<PreTokenizer>): PreTokenizer

export declare function sequenceProcessing(processors: Array<Processor>): Processor

export declare function slice(
  s: string,
  beginIndex?: number | undefined | null,
  endIndex?: number | undefined | null,
): string

export declare function splitPreTokenizer(
  pattern: string,
  behavior: string,
  invert?: boolean | undefined | null,
): PreTokenizer

export declare function stripAccentsNormalizer(): Normalizer

export declare function stripDecoder(content: string, left: number, right: number): Decoder

export declare function stripNormalizer(
  left?: boolean | undefined | null,
  right?: boolean | undefined | null,
): Normalizer

export declare function templateProcessing(
  single: string,
  pair?: string | undefined | null,
  specialTokens?: Array<[string, number]> | undefined | null,
): Processor

export declare const enum TruncationDirection {
  Left = 'Left',
  Right = 'Right',
}

export interface TruncationOptions {
  maxLength?: number
  strategy?: TruncationStrategy
  direction?: string | TruncationDirection
  stride?: number
}

export declare const enum TruncationStrategy {
  LongestFirst = 'LongestFirst',
  OnlyFirst = 'OnlyFirst',
  OnlySecond = 'OnlySecond',
}

export interface UnigramOptions {
  unkId?: number
  byteFallback?: boolean
}

export declare function whitespacePreTokenizer(): PreTokenizer

export declare function whitespaceSplitPreTokenizer(): PreTokenizer

export interface WordLevelOptions {
  unkToken?: string
}

export declare function wordPieceDecoder(prefix?: string = '##', cleanup?: bool = true): Decoder

export interface WordPieceOptions {
  unkToken?: string
  continuingSubwordPrefix?: string
  maxInputCharsPerWord?: number
}
