1 2 3 4 5 6 7 8 9 10 11 12 13 14 | from evaluator import * DESCRIPTION = "This test case is designed to check if the model can print out the tokens in a AutoTokenizer's vocabulary." TAGS = ['explain', 'python'] question = """ How do I get all tokens in a AutoTokenizer? """ TestGetVocab = question >> LLMRun() >> SubstringEvaluator("get_vocab") if __name__ == "__main__": print(run_test(TestGetVocab)) |