mirror of
https://github.com/sheerun/vim-polyglot.git
synced 2025-11-08 11:33:52 -05:00
441 lines
14 KiB
Ruby
Executable File
441 lines
14 KiB
Ruby
Executable File
#!/usr/bin/env ruby
|
|
|
|
require 'open-uri'
|
|
require 'open3'
|
|
require 'yaml'
|
|
require 'fileutils'
|
|
require 'set'
|
|
require 'json'
|
|
|
|
Dir.chdir(File.dirname(__dir__))
|
|
|
|
PACKAGES = YAML.load_stream(File.read('packages.yaml'))
|
|
|
|
BASE_URL = 'https://raw.githubusercontent.com/github/linguist/master'
|
|
|
|
DIRS = {
|
|
default: %w(syntax indent doc compiler autoload ftplugin ctags after/syntax after/indent after/ftplugin),
|
|
all: %w(syntax indent compiler autoload ftplugin after extras ctags doc),
|
|
syntax: %w(syntax indent after/syntax after/indent)
|
|
}
|
|
|
|
def parallel(*procs)
|
|
threads = procs.map { |p| Thread.new { method(p).call } }
|
|
threads.map(&:join).map(&:value)
|
|
end
|
|
|
|
def read_strings(data, keys, print=false)
|
|
if data.is_a?(Hash)
|
|
data.flat_map do |key, val|
|
|
read_strings(val, keys, keys.include?(key))
|
|
end
|
|
elsif data.is_a?(Array)
|
|
data.flat_map { |d| read_strings(d, keys, print) }
|
|
elsif data.is_a?(String)
|
|
print ? [data] : []
|
|
else
|
|
[]
|
|
end
|
|
end
|
|
|
|
def transform_with(data, keys, transfrom=false, &block)
|
|
if data.is_a?(Hash)
|
|
Hash[data.map do |key, val|
|
|
[key, transform_with(val, keys, keys.include?(key), &block)]
|
|
end]
|
|
elsif data.is_a?(Array)
|
|
data.map { |d| transform_with(d, keys, transfrom, &block) }
|
|
elsif data.is_a?(String)
|
|
transfrom ? yield(data) : data
|
|
else
|
|
data
|
|
end
|
|
end
|
|
|
|
def each_hash(data, &block)
|
|
if data.is_a?(Hash)
|
|
yield data
|
|
data.each do |key, val|
|
|
each_hash(val, &block)
|
|
end
|
|
elsif data.is_a?(Array)
|
|
data.map { |d| each_hash(d, &block) }
|
|
end
|
|
end
|
|
|
|
def patterns_to_vim_patterns(patterns)
|
|
stdin, stdout, stderr = Open3.popen3('vim', '-V', '--clean', '/dev/stdin', '-es', '-c', "echo expand('%:p:h') | source #{__dir__}/scripts/eregex.vim", '-c', "for line in range(0, line('$')) | call setline(line, ExtendedRegex2VimRegex(getline(line))) | endfor", '-c', ':wq! /dev/stdout', chdir: __dir__)
|
|
stdin.write(patterns.join("\n"))
|
|
stdin.close
|
|
stdout.readlines
|
|
end
|
|
|
|
def transform_patterns(data)
|
|
patterns = read_strings(data, ["pattern", "patterns"])
|
|
patterns_mapping = Hash[patterns.zip(patterns_to_vim_patterns(patterns))]
|
|
transform_with(data, ["pattern", "patterns"]) { |a| patterns_mapping[a] }
|
|
end
|
|
|
|
def load_heuristics
|
|
url = "#{BASE_URL}/lib/linguist/heuristics.yml"
|
|
data = URI.open(url) { |io| YAML.load(io.read) }
|
|
each_hash(data["disambiguations"]) do |h|
|
|
if h.has_key?("named_pattern")
|
|
h["pattern"] = data["named_patterns"].fetch(h["named_pattern"])
|
|
h.delete("named_pattern")
|
|
end
|
|
end
|
|
transform_patterns(data["disambiguations"])
|
|
end
|
|
|
|
def load_languages
|
|
url = "#{BASE_URL}/lib/linguist/languages.yml"
|
|
data = URI.open(url) { |io| YAML.load(io.read) }
|
|
end
|
|
|
|
def parse_remote(remote)
|
|
match = remote.match(/(?<repo>[^@:]+)(?:@(?<branch>[^:]+))?(?::(?<path>.*))?/)
|
|
[match[:repo], match[:branch] || "master", match[:path]]
|
|
end
|
|
|
|
def copy_file(package, src, dest)
|
|
return unless [".vim", ".ctags", ".vital", ".txt"].include?(File.extname(src))
|
|
FileUtils.mkdir_p(File.dirname(dest))
|
|
name = package.fetch("name")
|
|
|
|
open(src, "r") do |input|
|
|
open(dest, "a+") do |output|
|
|
if name == "jsx"
|
|
output << "if !exists('g:polyglot_disabled') || !(index(g:polyglot_disabled, 'typescript') != -1 || index(g:polyglot_disabled, 'typescript') != -1 || index(g:polyglot_disabled, 'jsx') != -1)\n\n"
|
|
else
|
|
output << "if !exists('g:polyglot_disabled') || index(g:polyglot_disabled, '#{name}') == -1\n\n"
|
|
end
|
|
IO.copy_stream(input, output)
|
|
output << "\nendif\n"
|
|
end
|
|
end
|
|
end
|
|
|
|
def download
|
|
FileUtils.rm_rf('tmp')
|
|
|
|
PACKAGES.each_slice(20) do |batch|
|
|
batch.map do |package|
|
|
Thread.new do
|
|
repo, branch, path = parse_remote(package.fetch("remote"))
|
|
dir = "tmp/" + repo.split('/')[1]
|
|
FileUtils.mkdir_p(dir)
|
|
url = "https://codeload.github.com/#{repo}/tar.gz/#{branch}"
|
|
`curl --silent -fL #{url} | tar -zx -C "#{dir}" --strip 1`
|
|
progress
|
|
end
|
|
end.map(&:join)
|
|
end
|
|
end
|
|
|
|
$i = 0
|
|
LYRICS = "Never gonna give you up. Never gonna let you down. " +
|
|
"Never gonna run around and desert you. " +
|
|
"Never gonna make you cry. Never gonna say goodbye. " +
|
|
"Never gonna tell a lie and hurt you."
|
|
|
|
$mutex = Mutex.new
|
|
def progress
|
|
$mutex.synchronize do
|
|
$stdout.write(LYRICS[$i] || ".")
|
|
$i += 1
|
|
end
|
|
end
|
|
|
|
def extract
|
|
FileUtils.rm_rf(DIRS[:all])
|
|
|
|
output = []
|
|
PACKAGES.map do |package|
|
|
repo, branch, path = parse_remote(package["remote"])
|
|
dir = "tmp/" + repo.split('/')[1]
|
|
subdirs = []
|
|
dirs = DIRS.fetch(package.fetch("dirs", "default").to_sym)
|
|
ignored_dirs = package.fetch("ignored_dirs", [])
|
|
if ignored_dirs.size > 0
|
|
dirs = dirs.reject { |d| ignored_dirs.any? { |id| d.start_with?(id) } }
|
|
end
|
|
dirs |= package.fetch("extra_dirs", [])
|
|
for subdir in dirs
|
|
subtree = "#{dir}/#{path ? path + "/" : ""}"
|
|
subpath = "#{subtree}#{subdir}"
|
|
if FileTest.directory?(subpath)
|
|
Dir.glob("#{subdir}/**/*", base: subtree).each do |p|
|
|
next unless File.file?("#{subtree}/#{p}")
|
|
copy_file(package, "#{subtree}/#{p}", p)
|
|
end
|
|
|
|
subdirs << subdir.split("/").last
|
|
elsif File.exist?(subpath)
|
|
copy_file(package, subpath, subdir)
|
|
end
|
|
end
|
|
|
|
output << "- [#{package["name"]}](https://github.com/#{repo}) (#{subdirs.uniq.join(", ")})"
|
|
progress
|
|
end
|
|
|
|
readme = File.read('README.md')
|
|
|
|
readme.gsub!(
|
|
%r{(?<=<!--Package Count-->).*?(?=<!--/Package Count-->)},
|
|
output.size.to_s
|
|
)
|
|
|
|
readme.gsub!(
|
|
%r{(?<=<!--Language Packs-->).*?(?=<!--/Language Packs-->)}m,
|
|
"\n" + output.sort.join("\n") + "\n"
|
|
)
|
|
|
|
File.write('README.md', readme)
|
|
end
|
|
|
|
def generate_ftdetect
|
|
heuristics, languages = parallel(:load_heuristics, :load_languages)
|
|
|
|
output = <<~EOS
|
|
" don't spam the user when Vim is started in Vi compatibility mode
|
|
let s:cpo_save = &cpo
|
|
set cpo&vim
|
|
|
|
if !exists('g:polyglot_disabled')
|
|
let g:polyglot_disabled = []
|
|
endif
|
|
|
|
function! s:SetDefault(name, value)
|
|
if !exists(a:name)
|
|
let {a:name} = a:value
|
|
endif
|
|
endfunction
|
|
|
|
call s:SetDefault('g:markdown_enable_spell_checking', 0)
|
|
call s:SetDefault('g:markdown_enable_input_abbreviations', 0)
|
|
call s:SetDefault('g:markdown_enable_mappings', 0)
|
|
|
|
" Enable jsx syntax by default
|
|
call s:SetDefault('g:jsx_ext_required', 0)
|
|
|
|
" Make csv loading faster
|
|
call s:SetDefault('g:csv_start', 1)
|
|
call s:SetDefault('g:csv_end', 2)
|
|
|
|
" Disable json concealing by default
|
|
call s:SetDefault('g:vim_json_syntax_conceal', 0)
|
|
|
|
call s:SetDefault('g:filetype_euphoria', 'elixir')
|
|
|
|
if !exists('g:python_highlight_all')
|
|
call s:SetDefault('g:python_highlight_builtins', 1)
|
|
call s:SetDefault('g:python_highlight_builtin_objs', 1)
|
|
call s:SetDefault('g:python_highlight_builtin_types', 1)
|
|
call s:SetDefault('g:python_highlight_builtin_funcs', 1)
|
|
call s:SetDefault('g:python_highlight_builtin_funcs_kwarg', 1)
|
|
call s:SetDefault('g:python_highlight_exceptions', 1)
|
|
call s:SetDefault('g:python_highlight_string_formatting', 1)
|
|
call s:SetDefault('g:python_highlight_string_format', 1)
|
|
call s:SetDefault('g:python_highlight_string_templates', 1)
|
|
call s:SetDefault('g:python_highlight_indent_errors', 1)
|
|
call s:SetDefault('g:python_highlight_space_errors', 1)
|
|
call s:SetDefault('g:python_highlight_doctests', 1)
|
|
call s:SetDefault('g:python_highlight_func_calls', 1)
|
|
call s:SetDefault('g:python_highlight_class_vars', 1)
|
|
call s:SetDefault('g:python_highlight_operators', 1)
|
|
call s:SetDefault('g:python_highlight_file_headers_as_comments', 1)
|
|
call s:SetDefault('g:python_slow_sync', 1)
|
|
endif
|
|
|
|
EOS
|
|
|
|
ambiguous_extensions = PACKAGES.flat_map { |e| e["filetypes"] }.flat_map do |e|
|
|
if e["linguist"]
|
|
((languages[e["linguist"]]["extensions"] || []).map { |e| e[1..-1] } | e.fetch("extra_extensions", [])) - e.fetch("ignored_extensions", [])
|
|
else
|
|
e.fetch("extensions", [])
|
|
end
|
|
end.group_by(&:itself).transform_values(&:count).select { |a, b| b > 1 }.keys.to_set
|
|
|
|
expected_filetypes = detect_filetypes
|
|
|
|
for package in PACKAGES
|
|
name = package.fetch("name")
|
|
output << if name == "jsx"
|
|
"if !(index(g:polyglot_disabled, 'typescript') != -1 || index(g:polyglot_disabled, 'typescript') != -1 || index(g:polyglot_disabled, 'jsx') != -1)\n"
|
|
else
|
|
"if index(g:polyglot_disabled, '#{name}') == -1\n"
|
|
end
|
|
|
|
filetypes = package["filetypes"] or raise "Unknown filetype for: #{package["name"]}"
|
|
|
|
for filetype in filetypes
|
|
name = filetype.fetch("name")
|
|
syntax = filetype["syntax"] ? " syntax=#{filetype["syntax"]}" : ""
|
|
set_command = package.fetch("custom_set", "set ft=#{name}#{syntax}")
|
|
|
|
linguist = filetype["linguist"] ? languages.fetch(filetype["linguist"]) : {}
|
|
extensions = filetype["extensions"] || linguist.fetch("extensions", []).map { |e| e[1..] }
|
|
extensions = (extensions | filetype.fetch("extra_extensions", [])) - filetype.fetch("ignored_extensions", [])
|
|
|
|
filenames = filetype["filenames"] || linguist.fetch("filenames", [])
|
|
filenames = (filenames | filetype.fetch("extra_filenames", [])) - filetype.fetch("ignored_filenames", [])
|
|
|
|
if expected_filetypes[name] && !filetype["syntax"]
|
|
for e in expected_filetypes.fetch(name)[:extensions] - extensions - expand_all(filetype.fetch("ignored_extensions", []))
|
|
puts "Probable missing extension for #{name}: #{e}"
|
|
end
|
|
|
|
for e in expand_all(expected_filetypes.fetch(name)[:filenames]) - expand_all(filenames).flat_map { |e| [e, e.gsub(/^\./, '')] } - expand_all(filetype.fetch("ignored_filenames", [])) - ['*']
|
|
puts "Probable missing filename for #{name}: #{e}"
|
|
end
|
|
end
|
|
|
|
for extension in extensions.sort
|
|
outer_filetype = filetype["outer_filetype"]
|
|
if outer_filetype
|
|
output << " au BufNewFile *.*.#{extension} execute \"do BufNewFile filetypedetect \" . expand(\"<afile>:r\") | #{outer_filetype}\n"
|
|
output << " au BufReadPre *.*.#{extension} execute \"do BufRead filetypedetect \" . expand(\"<afile>:r\") | #{outer_filetype}\n"
|
|
end
|
|
|
|
if ambiguous_extensions.include?(extension)
|
|
output << " au BufNewFile,BufRead *.#{extension} setf #{filetype["name"]}\n"
|
|
else
|
|
output << " au BufNewFile,BufRead *.#{extension} #{set_command}\n"
|
|
end
|
|
end
|
|
|
|
for filename in filenames.sort
|
|
if filename[0] == "."
|
|
filename = "{.,}" + filename[1..]
|
|
end
|
|
output << " au BufNewFile,BufRead #{filename} #{set_command}\n"
|
|
end
|
|
end
|
|
|
|
output << "endif\n\n"
|
|
end
|
|
|
|
output << <<~EOS
|
|
" restore Vi compatibility settings
|
|
let &cpo = s:cpo_save
|
|
unlet s:cpo_save
|
|
EOS
|
|
|
|
# filetypes = detect_filetypes
|
|
#
|
|
# for filetype in filetypes
|
|
#
|
|
# end
|
|
|
|
File.write('ftdetect/polyglot.vim', output)
|
|
end
|
|
|
|
def generate_tests
|
|
output = <<~EOS
|
|
function! TestFiletype(filetype)
|
|
try
|
|
enew
|
|
exec 'set ft=' . a:filetype
|
|
catch
|
|
echo 'Error loading filetype ' . a:filetype . ':'
|
|
echo v:exception
|
|
echo v:throwpoint
|
|
exec ':cq!'
|
|
endtry
|
|
endfunction
|
|
|
|
EOS
|
|
|
|
for package in PACKAGES
|
|
for filetype in package.fetch("filetypes", [])
|
|
output << "call TestFiletype('#{filetype["name"]}')\n"
|
|
end
|
|
end
|
|
|
|
File.write('scripts/test_filetypes.vim', output)
|
|
|
|
|
|
end
|
|
|
|
def brace_expansion(s)
|
|
r=1 # Dummy value to forward-declare the parse function `r`
|
|
t=->x{ # Function to parse a bracket block
|
|
x=x[0].gsub(/^{(.*)}$/){$1} # Remove outer brackets if both are present
|
|
# x[0] is required because of quirks in the `scan` function
|
|
x=x.scan(/(({(\g<1>|,)*}|[^,{}]|(?<=,|^)(?=,|$))+)/)
|
|
# Regex black magic: collect elements of outer bracket
|
|
x.map{|i|i=i[0];i[?{]?r[i]:i}.flatten # For each element with brackets, run parse function
|
|
}
|
|
r=->x{ # Function to parse bracket expansions a{b,c}{d,e}
|
|
i=x.scan(/({(\g<1>)*}|[^{} ]+)/) # Regex black magic: scan for adjacent sets of brackets
|
|
i=i.map(&t) # Map all elements against the bracket parser function `t`
|
|
i.shift.product(*i).map &:join # Combine the adjacent sets with cartesian product and join them together
|
|
}
|
|
s.split.map(&r).flatten
|
|
end
|
|
|
|
def square_expansion(s)
|
|
return [s] unless s.include?('[')
|
|
s.scan(/(\[[^\]]+\]|[^\[]+)/).map { |x| x[0] }
|
|
.map { |x| x[0] == "[" ? x[1..-2].split("") : [x] }
|
|
.reduce(&:product).map(&:flatten).map(&:join)
|
|
end
|
|
|
|
def comma_expanson(s)
|
|
s.scan(/{[^{]+}|[^{]+/).map { |a| a[0] == "{" ? a : a.split(",", -1) }.reduce([]) do |a, b|
|
|
a.size > 0 ?
|
|
(b.is_a?(String) ?
|
|
a[0..-2] + [a[-1] + b] :
|
|
a[0..-2] + [a[-1] + b[0]] + b[1..-1]) :
|
|
[b].flatten
|
|
end
|
|
end
|
|
|
|
def expand_all(pattern)
|
|
if pattern.is_a?(Array)
|
|
return pattern.flat_map { |p| expand_all(p) }
|
|
end
|
|
|
|
comma_expanson(pattern).flat_map do |e|
|
|
brace_expansion(e).flat_map do |e2|
|
|
square_expansion(e2)
|
|
end
|
|
end
|
|
end
|
|
|
|
def detect_filetypes
|
|
filetypes = Dir['tmp/*/ftdetect/*.vim'].flat_map do |file|
|
|
contents = File.read(file).gsub(/^\s*au(tocmd)?!?\s*$/, '')
|
|
results = contents.scan(/^\s*(?:au!|au|au[^g][^ ]*) +(?:\S+)\s+(\S+)[\s\\]+([^\n]+)/)
|
|
results = results.map do |a, b|
|
|
[
|
|
a,
|
|
b.gsub(/call (?:s:setf|s:StarSetf)\('([^']+)'\)/i, 'setf \1')
|
|
.gsub(/set(?:local)?\s+(?:ft|filetype)=(\S+)/, 'setf \1')
|
|
.gsub(/setf\S*/, 'setf')
|
|
.gsub(/.*setf\s+(\S+).*/, 'setf \1')
|
|
]
|
|
end.select { |a, b| b.match(/setf \S+/) }.map { |a, b| [a, b.split(" ")[1]] }
|
|
results
|
|
end
|
|
|
|
Hash[filetypes.flat_map do |ext, filetype|
|
|
expand_all(ext).map { |e| [filetype, e] }
|
|
end.group_by { |a, b| a }.map { |a, b| [a, b.map { |c, d| d }] }.map { |a, b|
|
|
[a, {
|
|
extensions: b.select { |x| x.match(/^\*\.[^\/]+$/) }.map { |a| a.strip[2..] },
|
|
filenames: b.select { |x| !x.match(/^\*\.[^\/]+$/) }
|
|
}]
|
|
}]
|
|
end
|
|
|
|
download
|
|
extract
|
|
generate_ftdetect
|
|
generate_tests
|
|
puts(" Bye! Have a wonderful time!")
|
|
FileUtils.rm_rf("tmp")
|