Some major updates including heuristics

- Allow to define heuristics in heuristics.yaml
- Migrate all filetypes from vim beginning with "a"
- Remove enhanced cpp syntax (it was too slow to load)
- Use setf instead of set ft for setting filetype (faster)
- Override native hauristics with au!
- Add globbing of files for packages
- Replace predefined dirs with extra_dirs and ignored_dirs
- Allow to define proper order of packages with topological sort
- Fix powershell detection
- Lint and fix many packages.yaml issues
- etc etd

scripts/build

@@ -6,18 +6,82 @@ require 'yaml'
 require 'fileutils'
 require 'set'
 require 'json'
+require 'tsort'
 
 Dir.chdir(File.dirname(__dir__))
 
-PACKAGES = YAML.load_stream(File.read('packages.yaml'))
-
 BASE_URL = 'https://raw.githubusercontent.com/github/linguist/master'
 
-DIRS = {
-  default: %w(syntax indent doc compiler autoload ftplugin ctags after/syntax after/indent after/ftplugin),
-  all: %w(syntax indent compiler autoload ftplugin after extras ctags doc),
-  syntax: %w(syntax indent after/syntax after/indent)
-}
+def camelize(str)
+  str.split(/[-_\.]/).map { |a| a.capitalize }.join("")
+end
+
+def except(hash, *keys)
+  h = hash.dup
+  keys.each { |k| h.delete(k) }
+  h
+end
+
+def load_data()
+  packages = Hash[YAML.load_stream(File.read('packages.yaml'))
+    .group_by { |a| a.fetch("name") }
+    .map { |a, b| [a, b.first] }]
+
+  deps = Hash.new { |h, k| h[k] = [] }
+
+  for package in packages.values
+    for name in [package.fetch("after", [])].flatten
+      packages[name] or raise "#{package["name"]} depends on unknown package: #{name}"
+      deps[name] << package["name"]
+    end
+  end
+
+  each_node = lambda {|&b| packages.keys.each(&b) }
+  each_child = lambda {|n, &b| deps[n].each(&b) }
+
+  languages = load_languages
+
+  # Reason can have ocaml as interpreter but let's not depend on it...
+  languages["Reason"]["interpreters"] -= ["ocaml"]
+
+  packages = TSort.tsort(each_node, each_child).map { |a| packages[a] }
+
+  for package in packages
+    for filetype in package["filetypes"]
+      if filetype["linguist"]
+        if filetype["extensions"]
+          raise "#{package["name"]} #{filetype["name"]}: extensions can't be set when linguist is defined"
+        end
+
+        if filetype["filenames"]
+          raise "#{package["name"]} #{filetype["name"]}: filenames can't be set when linguist is defined"
+        end
+
+        linguist = languages.fetch(filetype["linguist"])
+
+        filetype["extensions"] = (linguist["extensions"] || []).map { |e| e[1..-1] } |
+          filetype.fetch("extra_extensions", []) -
+          filetype.fetch("ignored_extensions", []).uniq
+
+        filetype["filenames"] = (linguist["filenames"] || []) |
+          filetype.fetch("extra_filenames", []) -
+          filetype.fetch("ignored_filenames", []).uniq
+        
+        filetype["interpreters"] = (linguist["interpreters"] || []) |
+          filetype.fetch("extra_interpreters", []) -
+          filetype.fetch("ignored_interpreters", []).uniq
+      else
+        filetype["extensions"] ||= []
+        filetype["filenames"] ||= []
+        filetype["interpreters"] ||= []
+      end
+    end
+  end
+
+  heuristics = YAML.load_stream(File.read('heuristics.yaml'))
+
+  [packages, transform_patterns(heuristics)]
+end
 
 def parallel(*procs)
   threads = procs.map { |p| Thread.new { method(p).call } }
@@ -38,17 +102,12 @@ def read_strings(data, keys, print=false)
   end
 end
 
-def transform_with(data, keys, transfrom=false, &block)
-  if data.is_a?(Hash)
-    Hash[data.map do |key, val|
-      [key, transform_with(val, keys, keys.include?(key), &block)]
-    end]
-  elsif data.is_a?(Array)
-    data.map { |d| transform_with(d, keys, transfrom, &block) }
-  elsif data.is_a?(String)
-    transfrom ? yield(data) : data
-  else
-    data
+def patterns_to_vim_patterns(patterns)
+  stdin, stdout, stderr = Open3.popen3('vim', '-V', '--clean', '/dev/stdin', '-es', '-c', "echo expand('%:p:h') | source #{__dir__}/eregex.vim", '-c', "for line in range(0, line('$')) | call setline(line, ExtendedRegex2VimRegex(getline(line))) | endfor", '-c', ':wq! /dev/stdout', chdir: __dir__)
+  stdin.write(patterns.join("\n"))
+  stdin.close
+  stdout.readlines.map(&:chomp).map do |r|
+    r.gsub('\b', '\(\<\|\>\)')
   end
 end
 
@@ -63,29 +122,21 @@ def each_hash(data, &block)
   end
 end
 
-def patterns_to_vim_patterns(patterns)
-  stdin, stdout, stderr = Open3.popen3('vim', '-V', '--clean', '/dev/stdin', '-es', '-c', "echo expand('%:p:h') | source #{__dir__}/scripts/eregex.vim", '-c', "for line in range(0, line('$')) | call setline(line, ExtendedRegex2VimRegex(getline(line))) | endfor", '-c', ':wq! /dev/stdout', chdir: __dir__)
-  stdin.write(patterns.join("\n"))
-  stdin.close
-  stdout.readlines
-end
 
-def transform_patterns(data)
-  patterns = read_strings(data, ["pattern", "patterns"])
-  patterns_mapping = Hash[patterns.zip(patterns_to_vim_patterns(patterns))]
-  transform_with(data, ["pattern", "patterns"]) { |a| patterns_mapping[a] }
-end
-
-def load_heuristics
-  url = "#{BASE_URL}/lib/linguist/heuristics.yml"
-  data = URI.open(url) { |io| YAML.load(io.read) }
-  each_hash(data["disambiguations"]) do |h|
-    if h.has_key?("named_pattern")
-      h["pattern"] = data["named_patterns"].fetch(h["named_pattern"])
-      h.delete("named_pattern")
+def transform_patterns(heuristics)
+  patterns = []
+  each_hash(heuristics) do |h|
+    if h.has_key?("pattern")
+      patterns << h["pattern"]
     end
   end
-  transform_patterns(data["disambiguations"])
+  patterns_mapping = Hash[patterns.zip(patterns_to_vim_patterns(patterns))]
+  each_hash(heuristics) do |h|
+    if h.has_key?("pattern")
+      h["pattern"] = patterns_mapping.fetch(h["pattern"])
+    end
+  end
+  heuristics
 end
 
 def load_languages
@@ -99,7 +150,6 @@ def parse_remote(remote)
 end
 
 def copy_file(package, src, dest)
-  return unless [".vim", ".ctags", ".vital", ".txt"].include?(File.extname(src))
   FileUtils.mkdir_p(File.dirname(dest))
   name = package.fetch("name")
 
@@ -116,17 +166,17 @@ def copy_file(package, src, dest)
   end
 end
 
-def download
-  FileUtils.rm_rf('tmp')
-
-  PACKAGES.each_slice(20) do |batch|
-    batch.map do |package|
+def download(packages)
+  packages.map { |p| p["remote"] or raise "No remote for: " + p["name"] }.uniq.each_slice(20) do |remotes|
+    remotes.map do |remote|
       Thread.new do
-        repo, branch, path = parse_remote(package.fetch("remote"))
-        dir = "tmp/" + repo.split('/')[1]
-        FileUtils.mkdir_p(dir)
-        url = "https://codeload.github.com/#{repo}/tar.gz/#{branch}"
-        `curl --silent -fL #{url} | tar -zx -C "#{dir}" --strip 1`
+        repo, branch, path = parse_remote(remote)
+        dir = "tmp/" + repo
+        unless File.exist?(dir)
+          FileUtils.mkdir_p(dir)
+          url = "https://codeload.github.com/#{repo}/tar.gz/#{branch}"
+          `curl --silent -fL #{url} | tar -zx -C "#{dir}" --strip 1`
+        end
         progress
       end
     end.map(&:join)
@@ -147,15 +197,116 @@ def progress
   end
 end
 
-def extract
-  FileUtils.rm_rf(DIRS[:all])
+def indent(str, amount)
+  str.gsub(/^(?!$)/, " " * amount).gsub(/\s+$/, "").gsub(/^ +\n/, "")
+end
+
+def pattern_to_condition(rule)
+  operator = (rule["negative"] ? "!" : "=") + "~" + (rule["ignore_case"] ? "?" : "#")
+
+  "line #{operator} '#{rule["pattern"]}'"
+end
+
+def rules_to_code(rules)
+  output = ""
+
+  vars = []
+  each_hash(rules) do |h|
+    if h.has_key?("set")
+      vars << h["set"]
+    end
+  end
+
+
+  if vars.size > 0
+    output << vars.uniq.sort.map do |var|
+      "let #{var} = 0"
+    end.join("\n") + "\n"
+  end
+
+  output << rule_to_code(rules)
+end
+
+def rule_to_code(rule)
+  if rule.has_key?("lines")
+    if rule["lines"] == 1
+      return <<~EOS
+        let line = getline(1)
+
+        #{indent(rule_to_code(except(rule, "lines")), 0)}
+      EOS
+    else
+      return <<~EOS
+        for lnum in range(1, min([line("$"), #{rule["lines"]}]))
+          let line = getline(lnum)
+
+        #{indent(rule_to_code(except(rule, "lines")), 2)}
+        endfor
+      EOS
+    end
+  end
+
+  if rule.has_key?("rules")
+    return rule["rules"].map { |r| indent(rule_to_code(r), 0) }.join("\n")
+  end
+
+  if rule.has_key?("pattern")
+    return <<~EOS
+      if #{pattern_to_condition(rule)}
+      #{indent(rule_to_code(except(rule, "pattern", "ignore_case", "negative")), 2)}
+      endif
+    EOS
+  end
+
+  if rule.has_key?("if_set")
+    return <<~EOS
+      if #{rule["negative"] ? "!" : ""}#{rule["if_set"]}
+      #{indent(rule_to_code(except(rule, "if_set", "negative")), 2)}
+      endif
+    EOS
+  end
+
+  if rule.has_key?("set")
+    return <<~EOS
+      let #{rule["set"]} = 1
+      #{indent(rule_to_code(except(rule, "set")), 0)}
+    EOS
+  end
+
+  if (rule.keys - ["filetype", "override", "set"]).size > 0
+    raise "Unknown rule: #{JSON.generate(rule)}"
+  end
+  
+  if rule.has_key?("override")
+    return <<~EOS
+      if exists("#{rule["override"]}")
+        exe "setf " . #{rule["override"]} | return
+      endif
+    EOS
+  end
+
+  if rule.has_key?("filetype")
+    return "setf #{rule["filetype"]} | return"
+  end
+
+  return ""
+end
+
+def extract(packages)
+  all_dirs = %w(syntax indent doc compiler autoload ftplugin ctags extras after)
+
+  default_dirs = %w(
+    syntax indent doc compiler autoload ftplugin ctags extras
+    after/syntax after/indent after/ftplugin
+  )
+
+  FileUtils.rm_rf(all_dirs)
 
   output = []
-  PACKAGES.map do |package|
+  packages.map do |package|
     repo, branch, path = parse_remote(package["remote"])
-    dir = "tmp/" + repo.split('/')[1]
-    subdirs = []
-    dirs = DIRS.fetch(package.fetch("dirs", "default").to_sym)
+    dir = "tmp/" + repo
+    dirs = package.fetch("dirs", default_dirs)
     ignored_dirs = package.fetch("ignored_dirs", [])
     if ignored_dirs.size > 0
       dirs = dirs.reject { |d| ignored_dirs.any? { |id| d.start_with?(id) } }
@@ -165,18 +316,31 @@ def extract
       subtree = "#{dir}/#{path ? path + "/" : ""}"
       subpath = "#{subtree}#{subdir}"
       if FileTest.directory?(subpath)
-        Dir.glob("#{subdir}/**/*", base: subtree).each do |p|
+        if repo == "vim/vim" && (["glob", "globs"] & package.keys).size == 0
+          raise "Package from vim/vim should define glob or globs: #{package["name"]}"
+        end
+        glob = package.fetch("glob", package.fetch('globs', '**/*.{vim,ctags,vital,txt}'))
+        Dir.glob("#{subdir}/#{glob}", base: subtree).each do |p|
           next unless File.file?("#{subtree}/#{p}")
+          if p.include?("samba")
+            raise package["name"]
+          end
           copy_file(package, "#{subtree}/#{p}", p)
         end
-
-        subdirs << subdir.split("/").last
       elsif File.exist?(subpath)
         copy_file(package, subpath, subdir)
       end
     end
 
-    output << "- [#{package["name"]}](https://github.com/#{repo}) (#{subdirs.uniq.join(", ")})"
+    if branch != "master" || path
+      if path
+        output << "- [#{package["name"]}](https://github.com/#{repo}/tree/#{branch}/#{path})"
+      else
+        output << "- [#{package["name"]}](https://github.com/#{repo}/tree/#{branch})"
+      end
+    else
+      output << "- [#{package["name"]}](https://github.com/#{repo})"
+    end
     progress
   end
 
@@ -195,16 +359,23 @@ def extract
   File.write('README.md', readme)
 end
 
-def generate_ftdetect
-  heuristics, languages = parallel(:load_heuristics, :load_languages)
-
+def generate_ftdetect(packages, heuristics)
   output = <<~EOS
     " don't spam the user when Vim is started in Vi compatibility mode
     let s:cpo_save = &cpo
     set cpo&vim
 
-    if !exists('g:polyglot_disabled')
-      let g:polyglot_disabled = []
+    " Disable all native vim ftdetect
+    if exists('g:polyglot_test')
+      autocmd!
+    endif
+
+    let s:disabled_packages = {}
+
+    if exists('g:polyglot_disabled')
+      for pkg in g:polyglot_disabled
+        let s:disabled_packages[pkg] = 1
+      endfor
     endif
 
     function! s:SetDefault(name, value)
@@ -248,47 +419,55 @@ def generate_ftdetect
       call s:SetDefault('g:python_highlight_file_headers_as_comments', 1)
       call s:SetDefault('g:python_slow_sync', 1)
     endif
-
   EOS
 
-  ambiguous_extensions = PACKAGES.flat_map { |e| e["filetypes"] }.flat_map do |e|
-    if e["linguist"]
-      ((languages[e["linguist"]]["extensions"] || []).map { |e| e[1..-1] } | e.fetch("extra_extensions", [])) - e.fetch("ignored_extensions", [])
-    else
-      e.fetch("extensions", [])
+  extensions = Hash.new { |h, k| h[k] = [] }
+
+  for package in packages
+    for filetype in package["filetypes"]
+      for ext in filetype["extensions"]
+        extensions[ext] << filetype["name"]
+      end
     end
-  end.group_by(&:itself).transform_values(&:count).select { |a, b| b > 1 }.keys.to_set
+  end
+
+  ambiguous_extensions = extensions
+    .select { |a, b| b.uniq.size > 1 }.keys.sort
 
   expected_filetypes = detect_filetypes
 
-  for package in PACKAGES
+  for package in packages
     name = package.fetch("name")
-    output << if name == "jsx"
-      "if !(index(g:polyglot_disabled, 'typescript') != -1 || index(g:polyglot_disabled, 'typescript') != -1 || index(g:polyglot_disabled, 'jsx') != -1)\n"
-    else
-      "if index(g:polyglot_disabled, '#{name}') == -1\n"
-    end
+
+    output << "if !has_key(s:disabled_packages, '#{name}')\n"
 
     filetypes = package["filetypes"] or raise "Unknown filetype for: #{package["name"]}"
 
+    package_heuristics = []
+
     for filetype in filetypes
       name = filetype.fetch("name")
-      syntax = filetype["syntax"] ? " syntax=#{filetype["syntax"]}" : ""
-      set_command = package.fetch("custom_set", "set ft=#{name}#{syntax}")
+      syntax = filetype["syntax"] ? " | set syntax=#{filetype["syntax"]}" : ""
 
-      linguist = filetype["linguist"] ? languages.fetch(filetype["linguist"]) : {}
-      extensions = filetype["extensions"] || linguist.fetch("extensions", []).map { |e| e[1..] }
-      extensions = (extensions | filetype.fetch("extra_extensions", [])) - filetype.fetch("ignored_extensions", [])
+      set_command = "setf #{name}"
 
-      filenames = filetype["filenames"] || linguist.fetch("filenames", [])
-      filenames = (filenames | filetype.fetch("extra_filenames", [])) - filetype.fetch("ignored_filenames", [])
+      if filetype["syntax"]
+        set_command = "if !did_filetype() | set ft=#{name} syntax=#{filetype["syntax"]} | endif"
+      end
+
+      if filetype["custom_set"]
+        set_command = filetype["custom_set"]
+      end
+
+      extensions = filetype["extensions"]
+      filenames = filetype["filenames"]
 
       if expected_filetypes[name] && !filetype["syntax"]
         for e in expected_filetypes.fetch(name)[:extensions] - extensions - expand_all(filetype.fetch("ignored_extensions", []))
           puts "Probable missing extension for #{name}: #{e}"
         end
 
-        for e in expand_all(expected_filetypes.fetch(name)[:filenames]) - expand_all(filenames).flat_map { |e| [e, e.gsub(/^\./, '')] } - expand_all(filetype.fetch("ignored_filenames", [])) - ['*']
+        for e in expected_filetypes.fetch(name)[:filenames] - expand_all(filenames).flat_map { |e| [e, e.gsub(/^\./, '')] } - expand_all(filetype.fetch("ignored_filenames", [])) - ['*']
           puts "Probable missing filename for #{name}: #{e}"
         end
       end
@@ -300,9 +479,14 @@ def generate_ftdetect
           output << "  au BufReadPre *.*.#{extension} execute \"do BufRead filetypedetect \" . expand(\"<afile>:r\") | #{outer_filetype}\n"
         end
 
-        if ambiguous_extensions.include?(extension)
-          output << "  au BufNewFile,BufRead *.#{extension} setf #{filetype["name"]}\n"
+        heuristic = heuristics.find { |h| h["extensions"].include?(extension) }
+        if heuristic
+          package_heuristics << heuristic
         else
+          # if ambiguous_extensions.include?(extension)
+          #   puts "Ambiguous extension without heuristic: #{extension} => #{filetype["name"]}"
+          # end
+          #
           output << "  au BufNewFile,BufRead *.#{extension} #{set_command}\n"
         end
       end
@@ -315,25 +499,114 @@ def generate_ftdetect
       end
     end
 
+    for heuristic in package_heuristics.uniq
+      extensions = heuristic["extensions"].map { |e| "*.#{e}" }
+      output << "  au! BufNewFile,BufRead #{extensions.join(",")} call polyglot#Detect#{camelize(heuristic["extensions"].first)}Filetype()\n"
+    end
+
     output << "endif\n\n"
   end
 
   output << <<~EOS
+    au BufNewFile,BufRead,StdinReadPost * 
+      \\ if !did_filetype() && expand("<amatch>") !~ g:ft_ignore_pat 
+      \\ | call polyglot#Heuristics() | endif
+
     " restore Vi compatibility settings
     let &cpo = s:cpo_save
     unlet s:cpo_save
   EOS
  
-  # filetypes = detect_filetypes
-  #
-  # for filetype in filetypes
-  #   
-  # end
-
   File.write('ftdetect/polyglot.vim', output)
+
+  output = <<~EOS
+    " Line continuation is used here, remove 'C' from 'cpoptions'
+    let s:cpo_save = &cpo
+    set cpo&vim
+
+    func! polyglot#Heuristics()
+      " Try to detect filetype from shebang
+      let l:filetype = polyglot#Shebang()
+      if l:filetype != ""
+        exec "setf " . l:filetype
+        return
+      endif
+    endfunc
+    
+    let s:interpreters = {
+    EOS
+
+    for filetype in packages.flat_map { |p| p.fetch("filetypes", []) }.sort_by { |a| a["name"] }
+      for interpreter in filetype["interpreters"]
+        output << "  \\ '#{interpreter}': '#{filetype["name"]}',\n"
+      end
+    end
+
+  output << <<~EOS
+      \\ }
+
+    let s:r_hashbang = '^#!\\s*\\(\\S\\+\\)\\s*\\(.*\\)\\s*'
+    let s:r_envflag = '%(\\S\\+=\\S\\+\\|-[iS]\\|--ignore-environment\\|--split-string\\)'
+    let s:r_env = '^\\%(\\' . s:r_envflag . '\\s\\+\\)*\\(\\S\\+\\)'
+
+    func! polyglot#Shebang()
+      let l:line1 = getline(1)
+
+      if l:line1 !~# "^#!"
+        return
+      endif
+
+      let l:pathrest = matchlist(l:line1, s:r_hashbang)
+
+      if len(l:pathrest) == 0
+        return 
+      endif
+
+      let [_, l:path, l:rest; __] = l:pathrest
+
+      let l:script = split(l:path, "/")[-1]
+
+      if l:script == "env"
+        let l:argspath = matchlist(l:rest, s:r_env)
+        if len(l:argspath) == 0
+          return
+        endif
+
+        let l:script = l:argspath[1]
+      endif
+
+      if has_key(s:interpreters, l:script)
+        return s:interpreters[l:script]
+      endif
+
+      for interpreter in keys(s:interpreters)
+        if l:script =~# '^' . interpreter
+          return s:interpreters[interpreter]
+        endif
+      endfor
+    endfunc
+
+  EOS
+
+  for heuristic in heuristics
+    output << <<~EOS
+      func! polyglot#Detect#{camelize(heuristic["extensions"].first)}Filetype()
+      #{indent(rules_to_code(heuristic), 2)}
+      endfunc
+
+    EOS
+  end
+
+  output << <<~EOS
+    " Restore 'cpoptions'
+    let &cpo = s:cpo_save
+    unlet s:cpo_save
+  EOS
+
+  File.write('autoload/polyglot.vim', output)
 end
 
-def generate_tests
+def generate_tests(packages)
   output = <<~EOS
     function! TestFiletype(filetype)
       try
@@ -349,7 +622,7 @@ def generate_tests
 
   EOS
 
-  for package in PACKAGES
+  for package in packages
     for filetype in package.fetch("filetypes", [])
       output << "call TestFiletype('#{filetype["name"]}')\n"
     end
@@ -407,7 +680,7 @@ def expand_all(pattern)
 end
 
 def detect_filetypes
-  filetypes = Dir['tmp/*/ftdetect/*.vim'].flat_map do |file|
+  filetypes = Dir['tmp/**/ftdetect/*.vim'].flat_map do |file|
     contents = File.read(file).gsub(/^\s*au(tocmd)?!?\s*$/, '')
     results = contents.scan(/^\s*(?:au!|au|au[^g][^ ]*) +(?:\S+)\s+(\S+)[\s\\]+([^\n]+)/)
     results = results.map do |a, b|
@@ -427,14 +700,24 @@ def detect_filetypes
   end.group_by { |a, b| a }.map { |a, b| [a, b.map { |c, d| d }] }.map { |a, b|
     [a, {
       extensions: b.select { |x| x.match(/^\*\.[^\/]+$/) }.map { |a| a.strip[2..] },
-      filenames: b.select { |x| !x.match(/^\*\.[^\/]+$/) }
+      filenames: expand_all(b.select { |x| !x.match(/^\*\.[^\/]+$/) })
     }]
   }]
 end
 
-download
-extract
-generate_ftdetect
-generate_tests
-puts(" Bye! Have a wonderful time!")
-FileUtils.rm_rf("tmp")
+if __FILE__ == $0
+  if !ENV["DEV"]
+    FileUtils.rm_rf("tmp")
+  end
+
+  packages, heuristics = load_data()
+  download(packages)
+  extract(packages)
+  generate_ftdetect(packages, heuristics)
+  generate_tests(packages)
+  puts(" Bye! Have a wonderful time!")
+
+  if !ENV["DEV"]
+    FileUtils.rm_rf("tmp")
+  end
+end

scripts/test

@@ -2,12 +2,10 @@
 
 set -e
 
-vim --clean -N --cmd "
-  filetype plugin indent on
-  syntax enable
+vim --clean -N -u <(echo "
   let &rtp='$PWD,'.&rtp
-  source ftdetect/polyglot.vim
+  let g:polyglot_test = 1
   source scripts/test_extensions.vim
-  source scripts/test_filetypes.vim
-  exec ':q!'
-"
+  \"source scripts/test_filetypes.vim
+  qa!
+")

scripts/test_extensions.vim

@@ -1,16 +1,209 @@
 function! TestExtension(filetype, filename, content)
   try
-    exec "e " . a:filename
+    let g:message = ""
+    exec "noautocmd n " . a:filename
+    put =a:content
+    1delete _
+    filetype detect
     exec "if &filetype != '" . a:filetype . "' \nthrow &filetype\nendif"
+    exec ":bw!"
   catch
+    echo g:message
     echo 'Filename "' . a:filename  . '" does not resolve to extension "' . a:filetype . '"'
     echo '  instead received: "' . v:exception . '"'
     exec ':cq!'
   endtry
 endfunction
 
+call TestExtension('sh', 'bash1', "#!/bin/bash")
+call TestExtension('sh', 'bash2', "#! /bin/bash")
+call TestExtension('sh', 'bash3', "#! /bin/bash2.3")
+call TestExtension('sh', 'bash4', "#!/usr/bin/env bash")
+call TestExtension('sh', 'bash6', "#!/usr/bin/env -i -=split-string foo=bar bash -l foo")
+call TestExtension('sh', 'bash1', "#!/bin/bash")
+
+" Vim help file
+call TestExtension('help', $VIMRUNTIME . '/doc/foobar.txt', '')
+
+" Abaqus or Trasys
+call TestExtension('abaqus', 'foobar.inp', "*HEADING\nFoobar")
+call TestExtension('trasys', 'foobar.inp', "MSC PATRAN\n* foobar\nHEADER SURFACE DATA\nBSC ENCLO1")
+
+" 8th (Firth-derivative)
+call TestExtension('8th', 'foobar.8th', '')
+call TestExtension('8th', 'foobar.8th', '')
+
+" A-A-P recipe
+call TestExtension('aap', 'foobar.aap', '')
+
+
+" A2ps printing utility
+call TestExtension('a2ps', '/etc/a2ps.cfg', '')
+call TestExtension('a2ps', '/usr/local/etc/a2ps.cfg', '')
+call TestExtension('a2ps', '/etc/a2ps/foobar.cfg', '')
+call TestExtension('a2ps', '/usr/local/etc/a2ps/foobar.cfg', '')
+call TestExtension('a2ps', '/tmp/a2psrc', '')
+call TestExtension('a2ps', '/tmp/.a2psrc', '')
+
+" ABAB/4
+call TestExtension('abap', 'foobar.abap', '')
+
+" ABC music notation
+call TestExtension('abc', 'foobar.abc', '')
+
+" ABEL
+call TestExtension('abel', 'foobar.abl', '')
+
+" AceDB
+call TestExtension('acedb', 'foobar.wrm', '')
+
+" Ada (83, 9X, 95)
+call TestExtension('ada', 'foobar.adb', '')
+call TestExtension('ada', 'foobar.ads', '')
+call TestExtension('ada', 'foobar.ada', '')
+call TestExtension('ada', 'foobar.gpr', '')
+
+" AHDL
+call TestExtension('ahdl', 'foobar.tdf', '')
+
+" AIDL
+call TestExtension('aidl', 'foobar.aidl', '')
+
+" AMPL
+call TestExtension('ampl', 'foobar.run', '')
+
+" Ant
+call TestExtension('ant', 'build.xml', '')
+
+" Arduino
+call TestExtension('arduino', 'foobar.ino', '')
+call TestExtension('arduino', 'foobar.pde', '')
+
+" Apache config file
+call TestExtension('apache', '.htaccess', '')
+call TestExtension('apache', '/etc/httpd/foobar.conf', '')
+call TestExtension('apache', '/etc/apache2/sites-foobar/foobar.com', '')
+call TestExtension('apache', '/usr/local/etc/httpd/foobar.conf', '')
+call TestExtension('apache', '/usr/local/etc/apache2/sites-foobar/foobar.com', '')
+
+" XA65 MOS6510 cross assembler
+call TestExtension('a65', 'foobar.a65', '')
+
+" Applescript
+call TestExtension('applescript', 'foobar.scpt', '')
+
+" Applix ELF
+call TestExtension('elf', 'foobar.am', '')
+call TestExtension('automake', 'Makefile.am', '')
+call TestExtension('automake', 'makefile.am', '')
+
+" ALSA configuration
+call TestExtension('alsaconf', '.asoundrc', '')
+call TestExtension('alsaconf', '/usr/share/alsa/alsa.conf', '')
+call TestExtension('alsaconf', '/media/foo/usr/share/alsa/alsa.conf', '')
+call TestExtension('alsaconf', '/etc/asound.conf', '')
+call TestExtension('alsaconf', '/media/foo/etc/asound.conf', '')
+
+" Arc Macro Language
+call TestExtension('aml', 'foobar.aml', '')
+
+" APT config file
+call TestExtension('aptconf', 'apt.conf', '')
+call TestExtension('aptconf', '/root/.aptitude/config', '')
+call TestExtension('aptconf', '/etc/apt/apt.conf.d/foo_bar-12', '')
+call TestExtension('aptconf', '/etc/apt/apt.conf.d/foo_bar-12.conf', '')
+call TestExtension('', '/etc/apt/apt.conf.d/.gsdf', '')
+
+" Arch Inventory file
+call TestExtension('arch', '.arch-inventory', '')
+call TestExtension('arch', '=tagging-method', '')
+
+" ART*Enterprise (formerly ART-IM)
+call TestExtension('art', 'foobar.art', '')
+
+" AsciiDoc
+call TestExtension('asciidoc', 'foobar.asciidoc', '')
+call TestExtension('asciidoc', 'foobar.adoc', '')
+
+" ASN.1
+call TestExtension('asn', 'foobar.asn', '')
+call TestExtension('asn', 'foobar.asn1', '')
+
+" Active Server Pages (with Visual Basic Script)
+call TestExtension('aspvbs', 'foobar.asa', '')
+let g:filetype_asa = 'fizfuz'
+call TestExtension('fizfuz', 'foobar.asa', '')
+
+" Active Server Pages (with Perl or Visual Basic Script)
+call TestExtension('aspvbs', 'vbs.asp', "")
+call TestExtension('aspperl', 'perl.asp', "<Job ID=\"DropFiles\">\n<script language=\"PerlScript\">\n</script>\n</Job>")
+let g:filetype_asp = 'fizfuz'
+call TestExtension('fizfuz', 'fizfuz.asp', '')
+
+
+" Grub (must be before catch *.lst)
+call TestExtension('grub', '/boot/grub/menu.lst', '')
+call TestExtension('grub', '/media/foobar/boot/grub/menu.lst', '')
+call TestExtension('grub', '/boot/grub/grub.conf', '')
+call TestExtension('grub', '/media/foobar/boot/grub/grub.conf', '')
+call TestExtension('grub', '/etc/grub.conf', '')
+call TestExtension('grub', '/media/foobar/etc/grub.conf', '')
+
+" Assembly (all kinds)
+" *.lst is not pure assembly, it has two extra columns (address, byte codes)
+
+au BufNewFile,BufRead *.asm,*.[sS],*.[aA],*.mac,*.lst	call dist#ft#FTasm()
+
+" Macro (VAX)
+call TestExtension('vmasm', 'foobar.mar', '')
+
+" Atlas
+call TestExtension('atlas', 'foobar.atl', '')
+call TestExtension('atlas', 'foobar.as', '')
+
+" Autoit v3
+call TestExtension('autoit', 'foobar.au3', '')
+
+" Autohotkey
+call TestExtension('autohotkey', 'foobar.ahk', '')
+
+" Automake
+call TestExtension('automake', 'Makefile.am', '')
+call TestExtension('automake', 'makefile.am', '')
+call TestExtension('automake', 'GNUmakefile.am', '')
+
+" Autotest .at files are actually m4
+call TestExtension('m4', 'foobar.at', '')
+
+" Avenue
+call TestExtension('ave', 'foobar.ave', '')
+
+" Awk
+call TestExtension('awk', 'foobar.awk', '')
+
+" vim-polyglot only
 call TestExtension('blade', 'test.blade.php', '')
 call TestExtension('yaml.ansible', 'playbook.yml', '')
 call TestExtension('yaml.ansible', 'host_vars/foobar', '')
-call TestExtension('yaml.ansible', 'handlers.foo.yaml', '')
+call TestExtension('yaml.ansible', 'handlers.foobar.yaml', '')
 call TestExtension('yaml.ansible', 'requirements.yaml', '')
+call TestExtension('ps1xml', 'foobar.ps1xml', '')
+
+" .m extension
+call TestExtension('octave', 'matlab.m', '')
+call TestExtension('objc', 'objc.m', "\n\n  #import <Foundation/Foundation.h>")
+call TestExtension('octave', 'objc.m', "results_ub_times=zeros(2,2,M);\n%results pour la lower bound")
+call TestExtension('mma', 'mathematica.m', "newcase[ \"00003\" ];\n  (* Hello world *)")
+call TestExtension('murphi', 'murphi.m', "type\n  square: 1 .. 9")
+call TestExtension('murphi', 'murphi.m', "something\n--foobar")
+call TestExtension('octave', 'percentcomment.m', "hello world\n%foobar")
+call TestExtension('objc', 'comment.m', "\n/* Hello world */")
+let g:filetype_m = 'fizfuz'
+call TestExtension('fizfuz', 'fizfuz.m', '')
+
+" .fs extension
+call TestExtension('forth', 'empty.fs', '')
+call TestExtension('fsharp', 'fsharp.fs', "let myInt = 5")
+call TestExtension('glsl', 'glsl.fs', "//#version 120\nvoid main() {}")
+let g:filetype_fs = 'fizfuz'
+call TestExtension('fizfuz', 'fizfuz.fs', '')

scripts/test_filetypes.vim

@@ -10,12 +10,40 @@ function! TestFiletype(filetype)
   endtry
 endfunction
 
+call TestFiletype('8th')
+call TestFiletype('a2ps')
+call TestFiletype('a65')
+call TestFiletype('aap')
+call TestFiletype('abap')
+call TestFiletype('abaqus')
+call TestFiletype('abc')
+call TestFiletype('abel')
+call TestFiletype('acedb')
 call TestFiletype('asl')
+call TestFiletype('ada')
+call TestFiletype('ahdl')
+call TestFiletype('aidl')
+call TestFiletype('alsaconf')
+call TestFiletype('aml')
+call TestFiletype('ampl')
+call TestFiletype('ant')
+call TestFiletype('apache')
 call TestFiletype('apiblueprint')
 call TestFiletype('applescript')
+call TestFiletype('aptconf')
+call TestFiletype('arch')
 call TestFiletype('arduino')
+call TestFiletype('art')
 call TestFiletype('asciidoc')
 call TestFiletype('autohotkey')
+call TestFiletype('automake')
+call TestFiletype('asn')
+call TestFiletype('aspvbs')
+call TestFiletype('aspperl')
+call TestFiletype('atlas')
+call TestFiletype('autoit')
+call TestFiletype('ave')
+call TestFiletype('awk')
 call TestFiletype('c')
 call TestFiletype('cpp')
 call TestFiletype('caddyfile')
@@ -32,6 +60,7 @@ call TestFiletype('cucumber')
 call TestFiletype('cuesheet')
 call TestFiletype('dart')
 call TestFiletype('dhall')
+call TestFiletype('grub')
 call TestFiletype('d')
 call TestFiletype('dcov')
 call TestFiletype('dd')
@@ -39,6 +68,7 @@ call TestFiletype('ddoc')
 call TestFiletype('dsdl')
 call TestFiletype('Dockerfile')
 call TestFiletype('yaml.docker-compose')
+call TestFiletype('elf')
 call TestFiletype('elixir')
 call TestFiletype('eelixir')
 call TestFiletype('elm')
@@ -49,6 +79,7 @@ call TestFiletype('fennel')
 call TestFiletype('ferm')
 call TestFiletype('fish')
 call TestFiletype('fbs')
+call TestFiletype('forth')
 call TestFiletype('fsharp')
 call TestFiletype('gdscript3')
 call TestFiletype('gitconfig')
@@ -95,13 +126,16 @@ call TestFiletype('llvm')
 call TestFiletype('tablegen')
 call TestFiletype('log')
 call TestFiletype('lua')
+call TestFiletype('m4')
 call TestFiletype('mako')
+call TestFiletype('octave')
 call TestFiletype('mma')
 call TestFiletype('markdown')
 call TestFiletype('markdown.mdx')
 call TestFiletype('meson')
 call TestFiletype('dosini')
 call TestFiletype('moon')
+call TestFiletype('murphi')
 call TestFiletype('nginx')
 call TestFiletype('nim')
 call TestFiletype('nix')
@@ -115,17 +149,16 @@ call TestFiletype('ocamlbuild_tags')
 call TestFiletype('ocpbuild')
 call TestFiletype('ocpbuildroot')
 call TestFiletype('sexplib')
-call TestFiletype('octave')
 call TestFiletype('opencl')
 call TestFiletype('perl')
 call TestFiletype('sql')
 call TestFiletype('sql')
 call TestFiletype('cql')
-call TestFiletype('php')
 call TestFiletype('blade')
+call TestFiletype('php')
 call TestFiletype('plantuml')
 call TestFiletype('pony')
-call TestFiletype('powershell')
+call TestFiletype('ps1')
 call TestFiletype('ps1xml')
 call TestFiletype('proto')
 call TestFiletype('pug')
@@ -181,13 +214,16 @@ call TestFiletype('vala')
 call TestFiletype('vbnet')
 call TestFiletype('vcl')
 call TestFiletype('velocity')
+call TestFiletype('vmasm')
 call TestFiletype('vue')
 call TestFiletype('xdc')
 call TestFiletype('xml')
 call TestFiletype('xsl')
-call TestFiletype('yaml')
 call TestFiletype('yaml.ansible')
+call TestFiletype('yaml')
 call TestFiletype('helm')
+call TestFiletype('help')
 call TestFiletype('zephir')
-call TestFiletype('zig')
 call TestFiletype('zir')
+call TestFiletype('zig')
+call TestFiletype('trasys')