Deduplicate polyglot files

autoload/polyglot/detect.vim

@@ -1,3 +1,5 @@
+" DO NOT EDIT CODE BELOW, IT IS GENERATED WITH MAKEFILE
+
 func! polyglot#detect#Inp()
   let line = getline(nextnonblank(1))
   if line =~# '^\*'

autoload/polyglot/shebang.vim

@@ -1,5 +1,3 @@
-" Please do not edit this file directly, instead modify polyglot.vim or scripts/build
-
 func! polyglot#shebang#Detect()
   let ft = s:Filetype()
   if ft != ""
@@ -54,7 +52,7 @@ func! s:Filetype()
   endfor
 endfunc
 
-
+" DO NOT EDIT CODE BELOW, IT IS GENERATED WITH MAKEFILE
 
 let s:interpreters = {
   \ 'osascript': 'applescript',

autoload/polyglot/sleuth.vim

@@ -1,3 +1,5 @@
+" DO NOT EDIT CODE BELOW, IT IS GENERATED WITH MAKEFILE
+
 let s:globs = {
   \ '8th': '*.8th',
   \ 'Dockerfile': '*.dockerfile,*.dock,*.Dockerfile,Dockerfile,dockerfile,Dockerfile*',

ftdetect/polyglot.vim

@@ -97,6 +97,7 @@ endfunc
 
 augroup filetypedetect
 
+" DO NOT EDIT CODE BELOW, IT IS GENERATED WITH MAKEFILE
 
 if !has_key(s:disabled_packages, '8th')
   au! BufNewFile,BufRead *.8th set ft=8th
@@ -992,7 +993,7 @@ if !has_key(s:disabled_packages, 'sql')
   au! BufNewFile,BufRead *.bdy,*.ddl,*.fnc,*.pck,*.pkb,*.pks,*.plb,*.pls,*.plsql,*.prc,*.spc,*.sql,*.tpb,*.tps,*.trg,*.tyb,*.tyc,*.typ,*.vw set ft=sql
 endif
 
-
+" DO NOT EDIT CODE ABOVE, IT IS GENERATED WITH MAKEFILE
 
 au! BufNewFile,BufRead,StdinReadPost * if expand("<afile>:e") == "" |
   \ call polyglot#shebang#Detect() | endif

polyglot.vim

@@ -1,367 +0,0 @@
-""" autoload/polyglot/shebang.vim
-
-func! polyglot#shebang#Detect()
-  let ft = s:Filetype()
-  if ft != ""
-    let &ft = ft
-  endif
-
-  if &ft == ""
-    runtime! scripts.vim
-  endif
-
-  return &ft != ""
-endfunc
-
-let s:r_hashbang = '^#!\s*\(\S\+\)\s*\(.*\)\s*'
-let s:r_envflag = '%(\S\+=\S\+\|-[iS]\|--ignore-environment\|--split-string\)'
-let s:r_env = '^\%(\' . s:r_envflag . '\s\+\)*\(\S\+\)'
-
-func! s:Filetype()
-  let l:line1 = getline(1)
-
-  if l:line1 !~# "^#!"
-    return
-  endif
-
-  let l:pathrest = matchlist(l:line1, s:r_hashbang)
-
-  if len(l:pathrest) == 0
-    return
-  endif
-
-  let [_, l:path, l:rest; __] = l:pathrest
-
-  let l:script = split(l:path, "/")[-1]
-
-  if l:script == "env"
-    let l:argspath = matchlist(l:rest, s:r_env)
-    if len(l:argspath) == 0
-      return
-    endif
-
-    let l:script = l:argspath[1]
-  endif
-
-  if has_key(s:interpreters, l:script)
-    return s:interpreters[l:script]
-  endif
-
-  for interpreter in keys(s:interpreters)
-    if l:script =~# '^' . interpreter
-      return s:interpreters[interpreter]
-    endif
-  endfor
-endfunc
-
-""" ftdetect/polyglot.vim
-
-" don't spam the user when Vim is started in Vi compatibility mode
-let s:cpo_save = &cpo
-set cpo&vim
-
-" Disable all native vim ftdetect
-if exists('g:polyglot_test')
-  autocmd!
-endif
-
-func! s:Observe(fn)
-  let b:polyglot_observe = a:fn
-  augroup polyglot-observer
-    au! CursorHold,CursorHoldI,BufWritePost <buffer>
-      \ execute('if polyglot#' . b:polyglot_observe . '() | au! polyglot-observer | endif')
-  augroup END
-endfunc
-
-let s:disabled_packages = {}
-let s:new_polyglot_disabled = []
-
-if exists('g:polyglot_disabled')
-  for pkg in g:polyglot_disabled
-    let base = split(pkg, '\.')
-    if len(base) > 0
-      let s:disabled_packages[pkg] = 1
-      call add(s:new_polyglot_disabled, base[0]) 
-    endif
-  endfor
-else
-  let g:polyglot_disabled_not_set = 1
-endif
-
-function! s:SetDefault(name, value)
-  if !exists(a:name)
-    let {a:name} = a:value
-  endif
-endfunction
-
-call s:SetDefault('g:markdown_enable_spell_checking', 0)
-call s:SetDefault('g:markdown_enable_input_abbreviations', 0)
-call s:SetDefault('g:markdown_enable_mappings', 0)
-
-" Enable jsx syntax by default
-call s:SetDefault('g:jsx_ext_required', 0)
-
-" Needed for sql highlighting
-call s:SetDefault('g:javascript_sql_dialect', 'sql')
-
-" Make csv loading faster
-call s:SetDefault('g:csv_start', 1)
-call s:SetDefault('g:csv_end', 2)
-
-" Disable json concealing by default
-call s:SetDefault('g:vim_json_syntax_conceal', 0)
-
-call s:SetDefault('g:filetype_euphoria', 'elixir')
-
-if !exists('g:python_highlight_all')
-  call s:SetDefault('g:python_highlight_builtins', 1)
-  call s:SetDefault('g:python_highlight_builtin_objs', 1)
-  call s:SetDefault('g:python_highlight_builtin_types', 1)
-  call s:SetDefault('g:python_highlight_builtin_funcs', 1)
-  call s:SetDefault('g:python_highlight_builtin_funcs_kwarg', 1)
-  call s:SetDefault('g:python_highlight_exceptions', 1)
-  call s:SetDefault('g:python_highlight_string_formatting', 1)
-  call s:SetDefault('g:python_highlight_string_format', 1)
-  call s:SetDefault('g:python_highlight_string_templates', 1)
-  call s:SetDefault('g:python_highlight_indent_errors', 1)
-  call s:SetDefault('g:python_highlight_space_errors', 1)
-  call s:SetDefault('g:python_highlight_doctests', 1)
-  call s:SetDefault('g:python_highlight_func_calls', 1)
-  call s:SetDefault('g:python_highlight_class_vars', 1)
-  call s:SetDefault('g:python_highlight_operators', 1)
-  call s:SetDefault('g:python_highlight_file_headers_as_comments', 1)
-  call s:SetDefault('g:python_slow_sync', 1)
-endif
-
-" We need it because scripts.vim in vim uses "set ft=" which cannot be
-" overridden with setf (and we can't use set ft= so our scripts.vim work)
-func! s:Setf(ft)
-  if &filetype !~# '\<'.a:ft.'\>'
-    let &filetype = a:ft
-  endif
-endfunc
-
-" Function used for patterns that end in a star: don't set the filetype if the
-" file name matches ft_ignore_pat.
-" When using this, the entry should probably be further down below with the
-" other StarSetf() calls.
-func! s:StarSetf(ft)
-  if expand("<amatch>") !~ g:ft_ignore_pat && &filetype !~# '\<'.a:ft.'\>'
-    let &filetype = a:ft
-  endif
-endfunc
-
-augroup filetypedetect
-
-" scripts/build inserts here filetype detection autocommands
-
-au! BufNewFile,BufRead,StdinReadPost * if expand("<afile>:e") == "" |
-  \ call polyglot#shebang#Detect() | endif
-
-au BufEnter * if &ft == "" && expand("<afile>:e") == ""  |
-      \ call s:Observe('shebang#Detect') | endif
-
-augroup END
-
-if !has_key(s:disabled_packages, 'autoindent')
-  " Code below re-implements sleuth for vim-polyglot
-  let g:loaded_sleuth = 1
-  let g:loaded_foobar = 1
-
-  " Makes shiftwidth to be synchronized with tabstop by default
-  if &shiftwidth == &tabstop
-    let &shiftwidth = 0
-  endif
-
-  function! s:guess(lines) abort
-    let options = {}
-    let ccomment = 0
-    let podcomment = 0
-    let triplequote = 0
-    let backtick = 0
-    let xmlcomment = 0
-    let heredoc = ''
-    let minindent = 10
-    let spaces_minus_tabs = 0
-    let i = 0
-
-    for line in a:lines
-      let i += 1
-
-      if !len(line) || line =~# '^\W*$'
-        continue
-      endif
-
-      if line =~# '^\s*/\*'
-        let ccomment = 1
-      endif
-      if ccomment
-        if line =~# '\*/'
-          let ccomment = 0
-        endif
-        continue
-      endif
-
-      if line =~# '^=\w'
-        let podcomment = 1
-      endif
-      if podcomment
-        if line =~# '^=\%(end\|cut\)\>'
-          let podcomment = 0
-        endif
-        continue
-      endif
-
-      if triplequote
-        if line =~# '^[^"]*"""[^"]*$'
-          let triplequote = 0
-        endif
-        continue
-      elseif line =~# '^[^"]*"""[^"]*$'
-        let triplequote = 1
-      endif
-
-      if backtick
-        if line =~# '^[^`]*`[^`]*$'
-          let backtick = 0
-        endif
-        continue
-      elseif &filetype ==# 'go' && line =~# '^[^`]*`[^`]*$'
-        let backtick = 1
-      endif
-
-      if line =~# '^\s*<\!--'
-        let xmlcomment = 1
-      endif
-      if xmlcomment
-        if line =~# '-->'
-          let xmlcomment = 0
-        endif
-        continue
-      endif
-
-      " This is correct order because both "<<EOF" and "EOF" matches end
-      if heredoc != ''
-        if line =~# heredoc
-          let heredoc = ''
-        endif
-        continue
-      endif
-      let herematch = matchlist(line, '\C<<\W*\([A-Z]\+\)\s*$')
-      if len(herematch) > 0
-        let heredoc = herematch[1] . '$'
-      endif
-
-      let spaces_minus_tabs += line[0] == "\t" ? 1 : -1
-
-      if line[0] == "\t"
-        setlocal noexpandtab
-        let &l:shiftwidth=&tabstop
-        let b:sleuth_culprit .= ':' . i
-        return 1
-      elseif line[0] == " "
-        let indent = len(matchstr(line, '^ *'))
-        if (indent % 2 == 0 || indent % 3 == 0) && indent < minindent
-          let minindent = indent
-        endif
-      endif
-    endfor
-
-    if minindent < 10
-      setlocal expandtab
-      let &l:shiftwidth=minindent
-      let b:sleuth_culprit .= ':' . i
-      return 1
-    endif
-
-    return 0
-  endfunction
-
-  function! s:detect_indent() abort
-    if &buftype ==# 'help'
-      return
-    endif
-
-    let b:sleuth_culprit = expand("<afile>:p")
-    if s:guess(getline(1, 32))
-      return
-    endif
-    let pattern = polyglot#sleuth#GlobForFiletype(&filetype)
-    if len(pattern) == 0
-      return
-    endif
-    let pattern = '{' . pattern . ',.git,.svn,.hg}'
-    let dir = expand('%:p:h')
-    let level = 3
-    while isdirectory(dir) && dir !=# fnamemodify(dir, ':h') && level > 0
-      " Ignore files from homedir and root 
-      if dir == expand('~') || dir == '/'
-        unlet b:sleuth_culprit
-        return
-      endif
-      for neighbor in glob(dir . '/' . pattern, 0, 1)[0:level]
-        let b:sleuth_culprit = neighbor
-        " Do not consider directories above .git, .svn or .hg
-        if fnamemodify(neighbor, ":h:t")[0] == "."
-          let level = 0
-          continue
-        endif
-        if neighbor !=# expand('%:p') && filereadable(neighbor)
-          if s:guess(readfile(neighbor, '', 32))
-            return
-          endif
-        endif
-      endfor
-
-      let dir = fnamemodify(dir, ':h')
-      let level -= 1
-    endwhile
-
-    unlet b:sleuth_culprit
-  endfunction
-
-  setglobal smarttab
-
-  function! SleuthIndicator() abort
-    let sw = &shiftwidth ? &shiftwidth : &tabstop
-    if &expandtab
-      return 'sw='.sw
-    elseif &tabstop == sw
-      return 'ts='.&tabstop
-    else
-      return 'sw='.sw.',ts='.&tabstop
-    endif
-  endfunction
-
-  augroup polyglot-sleuth
-    au!
-    au FileType * call s:detect_indent()
-    au User Flags call Hoist('buffer', 5, 'SleuthIndicator')
-  augroup END
-
-  command! -bar -bang Sleuth call s:detect_indent()
-endif
-
-func! s:verify()
-  if exists("g:polyglot_disabled_not_set")
-    if exists("g:polyglot_disabled")
-      echohl WarningMsg
-      echo "vim-polyglot: g:polyglot_disabled should be defined before loading vim-polyglot"
-      echohl None
-    endif
-
-    unlet g:polyglot_disabled_not_set
-  endif
-endfunc
-
-au VimEnter * call s:verify()
-
-" Save polyglot_disabled without postfixes
-if exists('g:polyglot_disabled')
-  let g:polyglot_disabled = s:new_polyglot_disabled
-endif
-
-" restore Vi compatibility settings
-let &cpo = s:cpo_save
-
-""" end

scripts/build

@@ -12,14 +12,6 @@ Dir.chdir(File.dirname(__dir__))
 
 BASE_URL = 'https://raw.githubusercontent.com/github/linguist/master'
 
-def read_section(name)
-  section = File.read('polyglot.vim').split('""" ').find { |e| e.start_with?(name) }
-  if section.nil?
-    raise StandardError.new("Section not found: #{name}")
-  end
-  "\" Please do not edit this file directly, instead modify polyglot.vim or scripts/build\n" + section[(section.index("\n") + 1)..-1]
-end
-
 def camelize(str)
   str.split(/[-_\.]/).map { |a| a.capitalize }.join("")
 end
@@ -431,7 +423,7 @@ def rule_to_code(rule)
 end
 
 def extract(packages)
-  all_dirs = %w(syntax indent doc compiler autoload ftplugin ctags extras after)
+  all_dirs = %w(syntax indent doc compiler ftplugin ctags extras after)
 
   default_dirs = %w(
     syntax indent doc compiler autoload ftplugin ctags extras
@@ -440,6 +432,12 @@ def extract(packages)
 
   FileUtils.rm_rf(all_dirs)
 
+  for dir in Dir.glob("*", base: "autoload")
+    if dir != "polyglot"
+      FileUtils.rm_rf("autoload/" + dir)
+    end
+  end
+
   output = []
   packages.map do |package|
     repo, branch, path, dir = parse_remote(package["remote"])
@@ -585,14 +583,17 @@ def generate_ftdetect(packages, heuristics)
 
   show_warnings(all_filetypes, expected_filetypes)
 
-  ftdetect = read_section('ftdetect/polyglot.vim')
+  ftdetect = File.read('ftdetect/polyglot.vim')
+
+  starting = '" DO NOT EDIT CODE BELOW, IT IS GENERATED WITH MAKEFILE'
+  ending = '" DO NOT EDIT CODE ABOVE, IT IS GENERATED WITH MAKEFILE'
 
   File.write(
     'ftdetect/polyglot.vim',
-     ftdetect.gsub('" scripts/build inserts here filetype detection autocommands') { output }
+    ftdetect.gsub(/(?<=#{starting}\n)(.*)(?=#{ending})/m) { output }
   )
 
-  output = []
+  output = ["\" DO NOT EDIT CODE BELOW, IT IS GENERATED WITH MAKEFILE\n"]
 
   for heuristic in heuristics
     output << <<~EOS
@@ -616,8 +617,13 @@ def generate_ftdetect(packages, heuristics)
 
   output << "  \\ }"
 
-  autoload_script = read_section('autoload/polyglot/shebang.vim')
-  File.write('autoload/polyglot/shebang.vim', autoload_script + "\n\n" + output)
+  inject_code('autoload/polyglot/shebang.vim', output)
+end
+
+def inject_code(path, code)
+  header = "\" DO NOT EDIT CODE BELOW, IT IS GENERATED WITH MAKEFILE\n"
+  normal, generated = File.read(path).split(header)
+  File.write(path, [normal, generated].join(header))
 end
 
 def generate_tests(packages)
@@ -734,7 +740,11 @@ def detect_filetypes(glob)
 end
 
 def generate_plugins(packages)
-  output = "let s:globs = {\n"
+  output = <<~EOS
+    " DO NOT EDIT CODE BELOW, IT IS GENERATED WITH MAKEFILE
+
+    let s:globs = {
+  EOS
 
   patterns = Hash.new { |h, k| h[k] = [] }