lua*Packages: Consolidate separate setup hooks together

- Lua packages now consistently use LUA_PATH/LUA_CPATH rather than a mix
  of those and NIX_LUA_PATH/NIX_LUA_CPATH
- Lua libraries are now consistently only added to the search path
variables if:
    1) The library actually has a corresponding directory to search
    2) The library is not already present in the search path
  This should help prevent the search paths from growing overly large
- Fixed bugs in some path helpers
- Changed the affected shell script indentation to 2 spaces; nixpkgs
  shell scripts are inconsistently split between 2 and 4 space
  indentation, but 2 matches better with the Nix expressions, so IMO it
  makes more sense

5 files changed, 113 insertions, 142 deletions

diff --git a/pkgs/development/interpreters/lua-5/setup-hook.sh b/pkgs/development/interpreters/lua-5/setup-hook.sh
index 3989bedffdb..f7e56b62ac9 100644
--- a/pkgs/development/interpreters/lua-5/setup-hook.sh
+++ b/pkgs/development/interpreters/lua-5/setup-hook.sh
@@ -1,47 +1,47 @@
 # set -e
 
 nix_print() {
-    if [ ${NIX_DEBUG:-0} -ge $1 ]; then
-        echo "$2"
-    fi
+  if [ ${NIX_DEBUG:-0} -ge $1 ]; then
+    echo "$2"
+  fi
 }
 
 nix_debug() {
-    nix_print 3 "$1"
+  nix_print 3 "$1"
 }
 
 addToLuaSearchPathWithCustomDelimiter() {
-    local varName="$1"
-    local absPattern="$2"
-    # delete longest match starting from the lua placeholder '?'
-    local topDir="${absPattern%%\?*}"
+  local varName="$1"
+  local absPattern="$2"
+  # delete longest match starting from the lua placeholder '?'
+  local topDir="${absPattern%%\?*}"
 
-    # export only if the folder exists else LUA_PATH grows too big
-    if  [ ! -d "$topDir" ]; then return; fi
+  # export only if the folder exists else LUA_PATH/LUA_CPATH grow too large
+  if [[ ! -d "$topDir" ]]; then return; fi
 
-    export "${varName}=${!varName:+${!varName};}${absPattern}"
+  # export only if we haven't already got this dir in the search path
+  if [[ ${!varName} == *"$absPattern"* ]]; then return; fi
+
+  export "${varName}=${!varName:+${!varName};}${absPattern}"
 }
 
 addToLuaPath() {
-    local dir="$1"
-
-    if [[ ! -d "$dir" ]]; then
-        nix_debug "$dir not a directory abort"
-        return 0
-    fi
-    cd "$dir"
-    for pattern in @luapathsearchpaths@;
-    do
-        addToLuaSearchPathWithCustomDelimiter NIX_LUA_PATH "$PWD/$pattern"
-    done
-
-    # LUA_CPATH
-    for pattern in @luacpathsearchpaths@;
-    do
-        addToLuaSearchPathWithCustomDelimiter NIX_LUA_CPATH "$PWD/$pattern"
-    done
-    cd - >/dev/null
+  local dir="$1"
+
+  if [[ ! -d "$dir" ]]; then
+    nix_debug "$dir not a directory abort"
+    return 0
+  fi
+  cd "$dir"
+  for pattern in @luapathsearchpaths@; do
+    addToLuaSearchPathWithCustomDelimiter LUA_PATH "$PWD/$pattern"
+  done
+
+  # LUA_CPATH
+  for pattern in @luacpathsearchpaths@; do
+    addToLuaSearchPathWithCustomDelimiter LUA_CPATH "$PWD/$pattern"
+  done
+  cd - >/dev/null
 }
 
 addEnvHooks "$hostOffset" addToLuaPath
-
diff --git a/pkgs/development/interpreters/lua-5/wrap.sh b/pkgs/development/interpreters/lua-5/wrap.sh
index 545a0ae271c..5941ff4a6b9 100644
--- a/pkgs/development/interpreters/lua-5/wrap.sh
+++ b/pkgs/development/interpreters/lua-5/wrap.sh
@@ -4,73 +4,72 @@
 set -e
 
 wrapLuaPrograms() {
-    wrapLuaProgramsIn "$out/bin" "$out $luaPath"
+  wrapLuaProgramsIn "$out/bin" "$out $luaPath"
 }
 
 # Builds environment variables like LUA_PATH and PATH walking through closure
 # of dependencies.
 buildLuaPath() {
-    local luaPath="$1"
-    local path
-
-    # Create an empty table of paths (see doc on loadFromPropagatedInputs
-    # for how this is used). Build up the program_PATH and program_LUA_PATH
-    # variables.
-    declare -A luaPathsSeen=()
-    program_PATH=
-    luaPathsSeen["@lua@"]=1
-    addToSearchPath program_PATH @lua@/bin
-    for path in $luaPath; do
-        addToLuaPath "$path"
-    done
+  local luaPath="$1"
+  local path
+
+  # Create an empty table of paths (see doc on loadFromPropagatedInputs
+  # for how this is used). Build up the program_PATH and program_LUA_PATH
+  # variables.
+  declare -A luaPathsSeen=()
+  program_PATH=
+  luaPathsSeen["@lua@"]=1
+  addToSearchPath program_PATH @lua@/bin
+  for path in $luaPath; do
+    addToLuaPath "$path"
+  done
 }
 
-
 # with an executable shell script which will set some environment variables
 # and then call into the original binary (which has been given a .wrapped suffix).
 # luaPath is a list of directories
 wrapLuaProgramsIn() {
-    local dir="$1"
-    local luaPath="$2"
-    local f
-
-    buildLuaPath "$luaPath"
-
-    if [ ! -d "$dir" ]; then
-        nix_debug "$dir not a directory"
-        return
+  local dir="$1"
+  local luaPath="$2"
+  local f
+
+  buildLuaPath "$luaPath"
+
+  if [ ! -d "$dir" ]; then
+    nix_debug "$dir not a directory"
+    return
+  fi
+
+  nix_debug "wrapping programs in [$dir]"
+
+  # Find all regular files in the output directory that are executable.
+  find "$dir" -type f -perm -0100 -print0 | while read -d "" f; do
+    # Rewrite "#! .../env lua" to "#! /nix/store/.../lua".
+    # Strip suffix, like "3" or "2.7m" -- we don't have any choice on which
+    # Lua to use besides one with this hook anyway.
+    if head -n1 "$f" | grep -q '#!.*/env.*\(lua\)'; then
+      sed -i "$f" -e "1 s^.*/env[ ]*\(lua\)[^ ]*^#! @executable@^"
     fi
 
-    nix_debug "wrapping programs in [$dir]"
-
-    # Find all regular files in the output directory that are executable.
-    find "$dir" -type f -perm -0100 -print0 | while read -d "" f; do
-        # Rewrite "#! .../env lua" to "#! /nix/store/.../lua".
-        # Strip suffix, like "3" or "2.7m" -- we don't have any choice on which
-        # Lua to use besides one with this hook anyway.
-        if head -n1 "$f" | grep -q '#!.*/env.*\(lua\)'; then
-            sed -i "$f" -e "1 s^.*/env[ ]*\(lua\)[^ ]*^#! @executable@^"
-        fi
-
-        # wrapProgram creates the executable shell script described
-        # above. The script will set LUA_(C)PATH and PATH variables!
-        # (see pkgs/build-support/setup-hooks/make-wrapper.sh)
-        local -a wrap_args=("$f"
-            --prefix PATH ':' "$program_PATH"
-            --prefix LUA_PATH ';' "$NIX_LUA_PATH"
-            --prefix LUA_CPATH ';' "$NIX_LUA_CPATH"
-        )
-
-        # Add any additional arguments provided by makeWrapperArgs
-        # argument to buildLuaPackage.
-        # makeWrapperArgs
-        local -a user_args="($makeWrapperArgs)"
-        local -a wrapProgramArgs=("${wrap_args[@]}" "${user_args[@]}")
-
-        # see setup-hooks/make-wrapper.sh
-        wrapProgram "${wrapProgramArgs[@]}"
-
-    done
+    # wrapProgram creates the executable shell script described
+    # above. The script will set LUA_(C)PATH and PATH variables!
+    # (see pkgs/build-support/setup-hooks/make-wrapper.sh)
+    local -a wrap_args=("$f"
+      --prefix PATH ':' "$program_PATH"
+      --prefix LUA_PATH ';' "$LUA_PATH"
+      --prefix LUA_CPATH ';' "$LUA_CPATH"
+    )
+
+    # Add any additional arguments provided by makeWrapperArgs
+    # argument to buildLuaPackage.
+    # makeWrapperArgs
+    local -a user_args="($makeWrapperArgs)"
+    local -a wrapProgramArgs=("${wrap_args[@]}" "${user_args[@]}")
+
+    # see setup-hooks/make-wrapper.sh
+    wrapProgram "${wrapProgramArgs[@]}"
+
+  done
 }
 
 # Adds the lib and bin directories to the LUA_PATH and PATH variables,
@@ -78,22 +77,22 @@ wrapLuaProgramsIn() {
 # `propagated-native-build-inputs`, while avoiding duplicating paths by
 # flagging the directories it has visited in `luaPathsSeen`.
 loadFromPropagatedInputs() {
-    local dir="$1"
-    # Stop if we've already visited here.
-    if [ -n "${luaPathsSeen[$dir]}" ]; then
-        return;
-    fi
-    luaPathsSeen[$dir]=1
-
-    addToLuaPath "$dir"
-    addToSearchPath program_PATH $dir/bin
-
-    # Inspect the propagated inputs (if they exist) and recur on them.
-    local prop="$dir/nix-support/propagated-native-build-inputs"
-    if [ -e "$prop" ]; then
-        local new_path
-        for new_path in $(cat $prop); do
-            loadFromPropagatedInputs "$new_path"
-        done
-    fi
+  local dir="$1"
+  # Stop if we've already visited here.
+  if [ -n "${luaPathsSeen[$dir]}" ]; then
+    return
+  fi
+  luaPathsSeen[$dir]=1
+
+  addToLuaPath "$dir"
+  addToSearchPath program_PATH $dir/bin
+
+  # Inspect the propagated inputs (if they exist) and recur on them.
+  local prop="$dir/nix-support/propagated-native-build-inputs"
+  if [ -e "$prop" ]; then
+    local new_path
+    for new_path in $(cat $prop); do
+      loadFromPropagatedInputs "$new_path"
+    done
+  fi
 }
diff --git a/pkgs/development/interpreters/lua-5/wrapper.nix b/pkgs/development/interpreters/lua-5/wrapper.nix
index 14bd4a41646..816744e61cf 100644
--- a/pkgs/development/interpreters/lua-5/wrapper.nix
+++ b/pkgs/development/interpreters/lua-5/wrapper.nix
@@ -43,7 +43,7 @@ let
               rm -f "$out/bin/$prg"
               if [ -x "$prg" ]; then
                 nix_debug "Making wrapper $prg"
-                makeWrapper "$path/bin/$prg" "$out/bin/$prg" --suffix LUA_PATH ';' "$NIX_LUA_PATH"   --suffix LUA_CPATH ';' "$NIX_LUA_CPATH" ${stdenv.lib.concatStringsSep " " makeWrapperArgs}
+                makeWrapper "$path/bin/$prg" "$out/bin/$prg" --suffix LUA_PATH ';' "$LUA_PATH"   --suffix LUA_CPATH ';' "$LUA_CPATH" ${stdenv.lib.concatStringsSep " " makeWrapperArgs}
               fi
             fi
           done
diff --git a/pkgs/development/lua-modules/generic/default.nix b/pkgs/development/lua-modules/generic/default.nix
index 3dae32b5e15..71e1f7c05e6 100644
--- a/pkgs/development/lua-modules/generic/default.nix
+++ b/pkgs/development/lua-modules/generic/default.nix
@@ -1,6 +1,6 @@
 { lua, writeText, toLuaModule }:
 
-{ buildInputs ? [], disabled ? false, ... } @ attrs:
+{ disabled ? false, ... } @ attrs:
 
 if disabled then
   throw "${attrs.name} not supported by interpreter lua-${lua.luaversion}"
@@ -18,37 +18,8 @@ else
     //
     {
       name = "lua${lua.luaversion}-" + attrs.name;
-      buildInputs = buildInputs ++ [ lua ];
-
-      setupHook = writeText "setup-hook.sh" ''
-        # check for lua/clua modules and don't add duplicates
-
-        addLuaLibPath() {
-          local package_path="$1/share/lua/${lua.luaversion}"
-          if [[ ! -d $package_path ]]; then return; fi
-          if [[ $LUA_PATH = *"$package_path"* ]]; then return; fi
-
-          if [[ -z $LUA_PATH ]]; then
-            export LUA_PATH="$package_path/?.lua;$package_path/?/init.lua"
-          else
-            export LUA_PATH="$LUA_PATH;$package_path/?.lua;$package_path/?/init.lua"
-          fi
-        }
-
-        addLuaLibCPath() {
-          local package_cpath="$1/lib/lua/${lua.luaversion}"
-          if [[ ! -d $package_cpath ]]; then return; fi
-          if [[ $LUA_CPATH = *"$package_cpath"* ]]; then return; fi
-
-          if [[ -z $LUA_CPATH ]]; then
-            export LUA_CPATH="$package_cpath/?.so"
-          else
-            export LUA_CPATH="$LUA_CPATH;$package_cpath/?.so"
-          fi
-        }
-
-        addEnvHooks "$hostOffset" addLuaLibPath
-        addEnvHooks "$hostOffset" addLuaLibCPath
-      '';
+      propagatedBuildInputs = [
+        lua # propagate it for its setup-hook
+      ];
     }
   ) )
diff --git a/pkgs/top-level/lua-packages.nix b/pkgs/top-level/lua-packages.nix
index 2d34bdaabc0..c9e9305666e 100644
--- a/pkgs/top-level/lua-packages.nix
+++ b/pkgs/top-level/lua-packages.nix
@@ -64,17 +64,18 @@ in
 with self; {
 
   getLuaPathList = majorVersion: [
-     "lib/lua/${majorVersion}/?.lua" "share/lua/${majorVersion}/?.lua"
-    "share/lua/${majorVersion}/?/init.lua" "lib/lua/${majorVersion}/?/init.lua"
+    "share/lua/${majorVersion}/?.lua"
+    "share/lua/${majorVersion}/?/init.lua"
   ];
   getLuaCPathList = majorVersion: [
-     "lib/lua/${majorVersion}/?.so" "share/lua/${majorVersion}/?.so" "share/lua/${majorVersion}/?/init.so"
+    "lib/lua/${majorVersion}/?.so"
   ];
 
   # helper functions for dealing with LUA_PATH and LUA_CPATH
-  getPath       = lib : type : "${lib}/lib/lua/${lua.luaversion}/?.${type};${lib}/share/lua/${lua.luaversion}/?.${type}";
-  getLuaPath    = lib : getPath lib "lua";
-  getLuaCPath   = lib : getPath lib "so";
+  getPath = drv: pathListForVersion:
+    lib.concatMapStringsSep ";" (path: "${drv}/${path}") (pathListForVersion lua.luaversion);
+  getLuaPath = drv: getPath drv getLuaPathList;
+  getLuaCPath = drv: getPath drv getLuaCPathList;
 
   #define build lua package function
   buildLuaPackage = callPackage ../development/lua-modules/generic {