Tokenizerのリファクタ: `rest`の値を`Token::Rest`として`tokens`に含めるようにした #98

	name: Code quality check

	on:
	pull_request:
	paths:
	- '**.rs'
	- '**/Cargo.toml'

	jobs:
	build:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: Swatinem/rust-cache@v2
	with:
	cache-all-crates: true
	shared-key: 'code-quality'
	cache-provider: 'github'
	- name: Setup clippy
	run: rustup component add clippy
	- name: Code review with clippy
	uses: giraffate/clippy-action@v1
	with:
	reporter: 'github-pr-review'
	filter_mode: 'nofilter'
	github_token: ${{ secrets.GITHUB_TOKEN }}
	- name: Run benchmark
	uses: boa-dev/criterion-compare-action@v3
	with:
	token: ${{ secrets.GITHUB_TOKEN }}
	branchName: ${{ github.base_ref }}
	cwd: 'core'
	benchName: 'core_benchmark'

Provide feedback