TypeScript TokenizationRegistry.register方法代码示例

	test('Microsoft/monaco-editor#122: Unhandled Exception: TypeError: Unable to get property \'replace\' of undefined or null reference', () => {
		function assertViewLineTokens(model: Model, lineNumber: number, forceTokenization: boolean, expected: ViewLineToken[]): void {
			let actual = model.getLineTokens(lineNumber, !forceTokenization).inflate();
			assert.deepEqual(actual, expected);
		}

		let _tokenId = 10;
		const LANG_ID1 = 'indicisiveMode1';
		const LANG_ID2 = 'indicisiveMode2';
		const languageIdentifier1 = new LanguageIdentifier(LANG_ID1, 3);
		const languageIdentifier2 = new LanguageIdentifier(LANG_ID2, 4);

		const tokenizationSupport: ITokenizationSupport = {
			getInitialState: () => NULL_STATE,
			tokenize: undefined,
			tokenize2: (line, state) => {
				let myId = ++_tokenId;
				let tokens = new Uint32Array(2);
				tokens[0] = 0;
				tokens[1] = (
					myId << MetadataConsts.FOREGROUND_OFFSET
				) >>> 0;
				return new TokenizationResult2(tokens, state);
			}
		};

		let registration1 = TokenizationRegistry.register(LANG_ID1, tokenizationSupport);
		let registration2 = TokenizationRegistry.register(LANG_ID2, tokenizationSupport);

		let model = Model.createFromString('A model with\ntwo lines');

		assertViewLineTokens(model, 1, true, [new ViewLineToken(12, 'mtk1')]);
		assertViewLineTokens(model, 2, true, [new ViewLineToken(9, 'mtk1')]);

		model.setMode(languageIdentifier1);

		assertViewLineTokens(model, 1, true, [new ViewLineToken(12, 'mtk11')]);
		assertViewLineTokens(model, 2, true, [new ViewLineToken(9, 'mtk12')]);

		model.setMode(languageIdentifier2);

		assertViewLineTokens(model, 1, false, [new ViewLineToken(12, 'mtk1')]);
		assertViewLineTokens(model, 2, false, [new ViewLineToken(9, 'mtk1')]);

		model.dispose();
		registration1.dispose();
		registration2.dispose();
	});

		constructor() {
			super(OUTER_LANGUAGE_ID);
			this._register(LanguageConfigurationRegistry.register(this.getLanguageIdentifier(), {}));

			this._register(TokenizationRegistry.register(this.getLanguageIdentifier().language, {
				getInitialState: (): IState => NULL_STATE,
				tokenize: undefined,
				tokenize2: (line: string, state: IState): TokenizationResult2 => {
					const tokensArr: number[] = [];
					let prevLanguageId: LanguageIdentifier | undefined = undefined;
					for (let i = 0; i < line.length; i++) {
						const languageId = (line.charAt(i) === 'x' ? INNER_LANGUAGE_ID : OUTER_LANGUAGE_ID);
						if (prevLanguageId !== languageId) {
							tokensArr.push(i);
							tokensArr.push((languageId.id << MetadataConsts.LANGUAGEID_OFFSET));
						}
						prevLanguageId = languageId;
					}

					const tokens = new Uint32Array(tokensArr.length);
					for (let i = 0; i < tokens.length; i++) {
						tokens[i] = tokensArr[i];
					}
					return new TokenizationResult2(tokens, state);
				}
			}));
		}

	constructor(tokenType:string) {
		super();

		TokenizationRegistry.register(this.getId(), new TokenizationSupport(null, this.getId(), {
			getInitialState: () => new StateForMockTokenizingMode(this.getId(), tokenType)
		}, false));
	}

export function setTokensProvider(languageId: string, provider: TokensProvider | EncodedTokensProvider): IDisposable {
	let languageIdentifier = StaticServices.modeService.get().getLanguageIdentifier(languageId);
	if (!languageIdentifier) {
		throw new Error(`Cannot set tokens provider for unknown language ${languageId}`);
	}
	let adapter: modes.ITokenizationSupport;
	if (isEncodedTokensProvider(provider)) {
		adapter = new EncodedTokenizationSupport2Adapter(provider);
	} else {
		adapter = new TokenizationSupport2Adapter(StaticServices.standaloneThemeService.get(), languageIdentifier, provider);
	}
	return modes.TokenizationRegistry.register(languageId, adapter);
}

			constructor() {
				super();
				TokenizationRegistry.register(this.getId(), {
					getInitialState: () => {
						return new IndicisiveModeState();
					},
					tokenize: (line, state, offsetDelta, stopAtOffset) => {
						let myId = ++_tokenId;
						return {
							tokens: [new Token(0, 'custom.' + myId)],
							actualStopOffset: line.length,
							endState: state,
							modeTransitions: []
						};
					}
				});
			}

		constructor(commentsConfig: CommentRule) {
			super(OUTER_LANGUAGE_ID);
			this._register(LanguageConfigurationRegistry.register(this.getLanguageIdentifier(), {
				comments: commentsConfig
			}));

			this._register(modes.TokenizationRegistry.register(this.getLanguageIdentifier().language, {
				getInitialState: (): modes.IState => NULL_STATE,
				tokenize: undefined,
				tokenize2: (line: string, state: modes.IState): TokenizationResult2 => {
					let languageId = (/^  /.test(line) ? INNER_LANGUAGE_ID : OUTER_LANGUAGE_ID);

					let tokens = new Uint32Array(1 << 1);
					tokens[(0 << 1)] = 0;
					tokens[(0 << 1) + 1] = (
						(modes.ColorId.DefaultForeground << modes.MetadataConsts.FOREGROUND_OFFSET)
						| (languageId.id << modes.MetadataConsts.LANGUAGEID_OFFSET)
					);
					return new TokenizationResult2(tokens, state);
				}
			}));
		}

	constructor() {
		super();
		TokenizationRegistry.register(this.getId(), {
			getInitialState: (): IState => null,
			tokenize: (line: string, state: IState): ILineTokens => {
				let tokens: Token[] = [];
				for (let i = 0; i < line.length; i++) {
					let chr = line.charAt(i);
					let type = chr === '.' ? '' : 'text';
					if (tokens.length > 0 && tokens[tokens.length - 1].type === type) {
						continue;
					}
					tokens.push(new Token(i, type));
				}
				return {
					tokens: tokens,
					actualStopOffset: -1,
					endState: null,
					modeTransitions: null
				};
			}
		});
	}

	setup(() => {
		let _lineIndex = 0;
		const tokenizationSupport: modes.ITokenizationSupport = {
			getInitialState: () => NULL_STATE,
			tokenize: undefined,
			tokenize2: (line: string, state: modes.IState): TokenizationResult2 => {
				let tokens = _tokens[_lineIndex++];

				let result = new Uint32Array(2 * tokens.length);
				for (let i = 0; i < tokens.length; i++) {
					result[2 * i] = tokens[i].startIndex;
					result[2 * i + 1] = (
						tokens[i].value << modes.MetadataConsts.FOREGROUND_OFFSET
					);
				}
				return new TokenizationResult2(result, state);
			}
		};
		const LANGUAGE_ID = 'modelModeTest1';
		languageRegistration = modes.TokenizationRegistry.register(LANGUAGE_ID, tokenizationSupport);
		model = TextModel.createFromString(_text.join('\n'), undefined, new modes.LanguageIdentifier(LANGUAGE_ID, 0));
		// force tokenization
		model.forceTokenization(model.getLineCount());
	});

export function setMonarchTokensProvider(languageId: string, languageDef: IMonarchLanguage): IDisposable {
	let lexer = compile(languageId, languageDef);
	let adapter = createTokenizationSupport(StaticServices.modeService.get(), StaticServices.standaloneThemeService.get(), languageId, lexer);
	return modes.TokenizationRegistry.register(languageId, adapter);
}

	constructor() {
		super();
		TokenizationRegistry.register(this.getId(), new TokenizationSupport(null, this.getId(), {
			getInitialState: () => new State(this.getId())
		}, false));
	}