From 8371520b1c3b738b995427539ed676e541d8dddb Mon Sep 17 00:00:00 2001
From: Adrian Vladu <avladu@cloudbasesolutions.com>
Date: Wed, 8 Apr 2020 19:39:18 +0300
Subject: [PATCH 01/25] Added PS script to build cloudbase-init from sources

Change-Id: I3750d5777abf8d2f56d98c20232801ed647d1aab
---
 scripts/Readme.md                      |  38 ++++
 scripts/build_install_from_sources.ps1 | 252 +++++++++++++++++++++++++
 2 files changed, 290 insertions(+)
 create mode 100644 scripts/Readme.md
 create mode 100644 scripts/build_install_from_sources.ps1

diff --git a/scripts/Readme.md b/scripts/Readme.md
new file mode 100644
index 00000000..3d4165ce
--- /dev/null
+++ b/scripts/Readme.md
@@ -0,0 +1,38 @@
+### Requirements ###
+
+#### Clean Windows Server 2016 /2019 or Windows 10 install with latest updates
+
+#### Python 3.7 installed and added to path (with pip installed).
+    Download link: https://www.python.org/ftp/python/3.7.7/python-3.7.7-amd64.exe
+
+#### Visual Studio 2015 Community installed, with the following components:
+    - Programming Languages -> Visual C++ (all)
+    - Windows and Web Development -> Windows 8.1 (only Tools and Windows SDKs)
+
+    VS 2015 Download link (you need to be logged in to access it): https://download.my.visualstudio.com/db/en_visual_studio_community_2015_with_update_1_x86_x64_web_installer_8234321.exe
+    If you prefer to not build pywin32, Visual Studio 2017 / 2019 can be used too.
+    Reboot after VS installation is complete.
+
+#### All the Python packages will be built and installed using pip flags: "--no-binary :all:"
+
+## The full build of cloudbase-init (and dependencies) from sources takes around 10 minutes.
+
+#### How to run:
+
+
+```powershell
+.\build_install_from_sources.ps1 `
+    -CloudbaseInitRepoUrl "https://github.com/cloudbase/cloudbase-init" `
+    -PyWin32RepoUrl "https://github.com/mhammond/pywin32" `
+    -PyMiRepoUrl "https://github.com/cloudbase/PyMI" `
+    -BuildDir "build"
+
+```
+
+
+#### Workflow of the script:
+   - Download and set pip upper requirements from OpenStack
+   - Create / clean temporary build directory
+   - Build and install PyWin32 from sources
+   - Build and install PyMI from sources
+   - Build, install and create Cloudbase-Init binary from sources
diff --git a/scripts/build_install_from_sources.ps1 b/scripts/build_install_from_sources.ps1
new file mode 100644
index 00000000..752e3e52
--- /dev/null
+++ b/scripts/build_install_from_sources.ps1
@@ -0,0 +1,252 @@
+#ps1
+param(
+    [string]$CloudbaseInitRepoUrl="https://github.com/cloudbase/cloudbase-init",
+    [string]$PyWin32RepoUrl="https://github.com/mhammond/pywin32",
+    [string]$PyMiRepoUrl="https://github.com/cloudbase/PyMI",
+    [string]$BuildDir=""
+)
+
+$ErrorActionPreference = "Stop"
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+$scriptPath = Split-Path -Parent $MyInvocation.MyCommand.Definition
+
+$PIP_BUILD_NO_BINARIES_ARGS = "--no-binary :all:"
+
+function Run-CmdWithRetry {
+    param(
+        $command,
+        [int]$maxRetryCount=3,
+        [int]$retryInterval=1
+    )
+
+    $currErrorActionPreference = $ErrorActionPreference
+    $ErrorActionPreference = "Continue"
+
+    $retryCount = 0
+    while ($true) {
+        try {
+            & $command
+            break
+        } catch [System.Exception] {
+            $retryCount++
+            if ($retryCount -ge $maxRetryCount) {
+                $ErrorActionPreference = $currErrorActionPreference
+                throw
+            } else {
+                Write-Error $_.Exception
+                Start-Sleep $retryInterval
+            }
+        }
+    }
+
+    $ErrorActionPreference = $currErrorActionPreference
+}
+
+function Download-File {
+    param(
+        $url,
+        $dest
+    )
+
+    Write-Host "Downloading: $url"
+
+    $webClient = New-Object System.Net.webclient
+    Run-CmdWithRetry {
+        $webClient.DownloadFile($url, $dest)
+    }
+}
+
+function Set-VCVars {
+    param(
+        $Version="14.0",
+        $Platform="x86_amd64"
+    )
+
+    Write-Host "Setting Visual Studio version ${Version} environment variables"
+
+    Push-Location "$ENV:ProgramFiles (x86)\Microsoft Visual Studio ${Version}\VC\"
+    try {
+        cmd /c "vcvarsall.bat $platform & set" |
+            ForEach-Object {
+                if ($_ -match "=") {
+                    $v = $_.split("=")
+                    Set-Item -Force -Path "ENV:\$($v[0])" -Value "$($v[1])"
+              }
+            }
+    } finally {
+        Pop-Location
+    }
+}
+
+function Run-Command {
+    param(
+        $cmd,
+        $arguments,
+        $expectedExitCode = 0
+    )
+
+    Write-Host "Executing: $cmd $arguments"
+
+    $p = Start-Process -Wait -PassThru `
+        -NoNewWindow $cmd -ArgumentList $arguments
+    if ($p.ExitCode -ne $expectedExitCode) {
+        throw "$cmd failed with exit code: $($p.ExitCode)"
+    }
+}
+
+function Clone-Repo {
+    param(
+        $Url,
+        $Destination
+    )
+
+    Write-Host "Cloning ${Url} to ${Destination}"
+
+    Run-CmdWithRetry {
+        try {
+            Push-Location $BuildDir
+            git clone $Url $Destination
+            if ($LastExitCode) {
+                throw "git clone ${Url} ${Destination} failed"
+            }
+        } finally {
+            Pop-Location
+        }
+    }
+}
+
+function Install-PythonRequirements {
+    param([string]$SourcePath,
+          [switch]$BuildWithoutBinaries
+    )
+
+    Write-Host "Installing Python requirements from ${SourcePath}"
+
+    $extraPipArgs = ""
+    if ($BuildWithoutBinaries) {
+        $extraPipArgs = $PIP_BUILD_NO_BINARIES_ARGS
+    }
+
+    Run-CmdWithRetry {
+        try {
+            Push-Location (Join-Path $BuildDir $SourcePath)
+            Run-Command -Cmd "python" -Arguments @("-m", "pip", "install", $extraPipArgs, "-r", ".\requirements.txt")
+        } finally {
+            Pop-Location
+        }
+    }
+}
+
+function Install-PythonPackage {
+    param([string]$SourcePath,
+          [switch]$BuildWithoutBinaries
+    )
+
+    Write-Host "Installing Python package from ${SourcePath}"
+
+    $extraPipArgs = ""
+    if ($BuildWithoutBinaries) {
+        $extraPipArgs = $PIP_BUILD_NO_BINARIES_ARGS
+    }
+
+    Run-CmdWithRetry {
+        try {
+            Push-Location (Join-Path $BuildDir $SourcePath)
+            Run-Command -Cmd "python" -Arguments @("-m", "pip", "install", $extraPipArgs, ".")
+        } finally {
+            Pop-Location
+        }
+    }
+}
+
+function Prepare-BuildDir {
+    Write-Host "Creating / Cleaning up build directory ${BuildDir}"
+
+    if ($BuildDir -and (Test-Path $BuildDir)) {
+        Remove-Item -Recurse -Force $BuildDir
+    }
+    New-Item -Type Directory -Path $BuildDir -Force | Out-Null
+}
+
+function Setup-PythonPip {
+    $env:PIP_CONSTRAINT = ""
+    $env:PIP_NO_BINARY = ""
+
+    # Update pip. Not needed for latest version of Python 3.7.
+    # Download-File "https://bootstrap.pypa.io/get-pip.py" "${BuildDir}/get-pip.py"
+    # Run-CmdWithRetry {
+    #    python "${BuildDir}/get-pip.py"
+    #    if ($LastExitCode) {
+    #        throw "Failed to run 'python get-pip.py'"
+    #    }
+    #}
+
+    # Cloudbase-Init Python requirements should respect the OpenStack upper constraints
+    Clone-Repo "https://github.com/openstack/requirements" "requirements"
+    $constraintsFilePath = Join-Path $BuildDir "requirements/upper-constraints.txt"
+
+    $env:PIP_CONSTRAINT = $constraintsFilePath
+}
+
+function Install-PyWin32FromSource {
+    param($Url)
+
+    $sourcePath = "pywin32"
+
+    Clone-Repo $Url $sourceFolder
+    Install-PythonPackage -SourcePath $sourcePath -BuildWithoutBinaries
+}
+
+function Install-PyMI {
+    param($Url)
+
+    $sourcePath = "PyMI"
+
+    Clone-Repo $Url $sourcePath
+    Install-PythonRequirements -SourcePath $sourcePath -BuildWithoutBinaries
+    Install-PythonPackage -SourcePath $sourcePath -BuildWithoutBinaries
+}
+
+function Install-CloudbaseInit {
+    param($Url)
+
+    $sourcePath = "cloudbase-init"
+
+    Clone-Repo $Url $sourcePath
+    Install-PythonRequirements -SourcePath $sourcePath -BuildWithoutBinaries
+    Install-PythonPackage -SourcePath $sourcePath
+}
+
+### Main ###
+
+try {
+    $startDate = Get-Date
+    Write-Host "Cloudbase-Init build started."
+
+    # Make sure that BuildDir is created and cleaned up properly
+    if (!$BuildDir) {
+        $BuildDir = "build"
+    }
+    if (![System.IO.Path]::IsPathRooted($BuildDir)) {
+        $BuildDir = Join-Path $scriptPath $BuildDir
+    }
+    Prepare-BuildDir
+
+    # Make sure VS 2015 is used
+    Set-VCVars -Version "14.0"
+
+    # Setup pip upper requirements
+    Setup-PythonPip
+
+    # Install PyWin32 from source
+    Install-PyWin32FromSource $PyWin32RepoUrl
+
+    # PyMI setup can be skipped once the upstream version is published on pypi
+    Install-PyMI $PyMiRepoUrl
+
+    Install-CloudbaseInit $CloudbaseInitRepoUrl
+} finally {
+    $endDate = Get-Date
+    Write-Host "Cloudbase-Init build finished after $(($endDate - $StartDate).Minutes + 1) minutes."
+}
+

From 8d7f31b6023ef6388207af9abac2a4b0eb35b4a2 Mon Sep 17 00:00:00 2001
From: Adrian Vladu <avladu@cloudbasesolutions.com>
Date: Thu, 9 Apr 2020 14:26:34 +0300
Subject: [PATCH 02/25] Fixed empty args

Change-Id: Ia34a098908a81063c83a836fab0d3663a0a8d931
---
 scripts/build_install_from_sources.ps1 | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/scripts/build_install_from_sources.ps1 b/scripts/build_install_from_sources.ps1
index 752e3e52..3e7698b4 100644
--- a/scripts/build_install_from_sources.ps1
+++ b/scripts/build_install_from_sources.ps1
@@ -122,15 +122,16 @@ function Install-PythonRequirements {
 
     Write-Host "Installing Python requirements from ${SourcePath}"
 
-    $extraPipArgs = ""
+    $cmdArgs = @("-m", "pip", "install")
     if ($BuildWithoutBinaries) {
-        $extraPipArgs = $PIP_BUILD_NO_BINARIES_ARGS
+        $cmdArgs += $PIP_BUILD_NO_BINARIES_ARGS
     }
+    $cmdArgs += @("-r", ".\requirements.txt")
 
     Run-CmdWithRetry {
         try {
             Push-Location (Join-Path $BuildDir $SourcePath)
-            Run-Command -Cmd "python" -Arguments @("-m", "pip", "install", $extraPipArgs, "-r", ".\requirements.txt")
+            Run-Command -Cmd "python" -Arguments $cmdArgs
         } finally {
             Pop-Location
         }
@@ -144,15 +145,16 @@ function Install-PythonPackage {
 
     Write-Host "Installing Python package from ${SourcePath}"
 
-    $extraPipArgs = ""
+    $cmdArgs = @("-m", "pip", "install")
     if ($BuildWithoutBinaries) {
-        $extraPipArgs = $PIP_BUILD_NO_BINARIES_ARGS
+        $cmdArgs += $PIP_BUILD_NO_BINARIES_ARGS
     }
+    $cmdArgs += "."
 
     Run-CmdWithRetry {
         try {
             Push-Location (Join-Path $BuildDir $SourcePath)
-            Run-Command -Cmd "python" -Arguments @("-m", "pip", "install", $extraPipArgs, ".")
+            Run-Command -Cmd "python" -Arguments $cmdArgs
         } finally {
             Pop-Location
         }

From 905128368ba9d79a173a20472140bde11575679f Mon Sep 17 00:00:00 2001
From: Adrian Vladu <avladu@cloudbasesolutions.com>
Date: Thu, 9 Apr 2020 20:36:24 +0300
Subject: [PATCH 03/25] Hacks to make embed python work

Change-Id: If6c4e3eafab29983c30363495942d6a9d36a5f5e
---
 scripts/build_install_from_sources.ps1 | 126 +++++++++++++++++++++++--
 1 file changed, 120 insertions(+), 6 deletions(-)

diff --git a/scripts/build_install_from_sources.ps1 b/scripts/build_install_from_sources.ps1
index 3e7698b4..15a06a51 100644
--- a/scripts/build_install_from_sources.ps1
+++ b/scripts/build_install_from_sources.ps1
@@ -3,6 +3,9 @@ param(
     [string]$CloudbaseInitRepoUrl="https://github.com/cloudbase/cloudbase-init",
     [string]$PyWin32RepoUrl="https://github.com/mhammond/pywin32",
     [string]$PyMiRepoUrl="https://github.com/cloudbase/PyMI",
+    [string]$EmbeddedPythonVersion="3.7.7",
+    [string]$ComtypesUrl="https://github.com/enthought/comtypes",
+    [string]$SetuptoolsUrl="https://github.com/pypa/setuptools",
     [string]$BuildDir=""
 )
 
@@ -15,7 +18,7 @@ $PIP_BUILD_NO_BINARIES_ARGS = "--no-binary :all:"
 function Run-CmdWithRetry {
     param(
         $command,
-        [int]$maxRetryCount=3,
+        [int]$maxRetryCount=1,
         [int]$retryInterval=1
     )
 
@@ -145,7 +148,7 @@ function Install-PythonPackage {
 
     Write-Host "Installing Python package from ${SourcePath}"
 
-    $cmdArgs = @("-m", "pip", "install")
+    $cmdArgs = @("-W ignore", "-m", "pip", "install")
     if ($BuildWithoutBinaries) {
         $cmdArgs += $PIP_BUILD_NO_BINARIES_ARGS
     }
@@ -161,6 +164,39 @@ function Install-PythonPackage {
     }
 }
 
+function Expand-Archive {
+    param([string]$archive,
+          [string]$outputDir
+    )
+
+    Push-Location $outputDir
+    try {
+        & "C:\Program Files\7-Zip\7z.exe" x -y $archive
+        if ($LastExitCode) {
+            throw "7z.exe failed on archive: $archive"
+        }
+    } finally {
+        Pop-Location
+    }
+}
+
+function Setup-PythonPackage {
+    param([string]$SourcePath)
+
+    Write-Host "Setup Python package from ${SourcePath}"
+
+    $cmdArgs = @("-W ignore", "setup.py", "install")
+
+    Run-CmdWithRetry {
+        try {
+            Push-Location (Join-Path $BuildDir $SourcePath)
+            Run-Command -Cmd "python" -Arguments $cmdArgs
+        } finally {
+            Pop-Location
+        }
+    }
+}
+
 function Prepare-BuildDir {
     Write-Host "Creating / Cleaning up build directory ${BuildDir}"
 
@@ -196,7 +232,16 @@ function Install-PyWin32FromSource {
     $sourcePath = "pywin32"
 
     Clone-Repo $Url $sourceFolder
-    Install-PythonPackage -SourcePath $sourcePath -BuildWithoutBinaries
+    Install-PythonPackage -SourcePath $sourcePath
+}
+
+function Install-ComtypesFromSource {
+    param($Url)
+
+    $sourcePath = "comptype"
+
+    Clone-Repo $Url $sourcePath
+    Install-PythonPackage -SourcePath $sourcePath
 }
 
 function Install-PyMI {
@@ -206,7 +251,7 @@ function Install-PyMI {
 
     Clone-Repo $Url $sourcePath
     Install-PythonRequirements -SourcePath $sourcePath -BuildWithoutBinaries
-    Install-PythonPackage -SourcePath $sourcePath -BuildWithoutBinaries
+    Install-PythonPackage -SourcePath $sourcePath
 }
 
 function Install-CloudbaseInit {
@@ -219,6 +264,68 @@ function Install-CloudbaseInit {
     Install-PythonPackage -SourcePath $sourcePath
 }
 
+function Install-SetuptoolsFromSource {
+    param($Url)
+
+    $sourcePath = "setuptools"
+
+    Clone-Repo $Url $sourcePath
+
+    Run-CmdWithRetry {
+        try {
+            Push-Location (Join-Path $BuildDir $SourcePath)
+            Run-Command -Cmd "python" -Arguments @("-W ignore", ".\bootstrap.py")
+            Run-Command -Cmd "python" -Arguments @("-W ignore", "setup.py", "install")
+            Run-Command -Cmd "python" -Arguments @("-W ignore", "-m", "easy_install", "pip")
+            Run-Command -Cmd "python" -Arguments @("-W ignore", "-m", "easy_install", "wheel")
+        } finally {
+            Pop-Location
+        }
+    }
+}
+
+function Setup-EmbeddedPythonEnvironment {
+    param($EmbeddedPythonVersion)
+
+    $EmbeddedPythonUrl = "https://www.python.org/ftp/python/${EmbeddedPythonVersion}/python-${EmbeddedPythonVersion}-embed-amd64.zip"
+    $SourcePythonUrl = "https://www.python.org/ftp/python/${EmbeddedPythonVersion}/Python-${EmbeddedPythonVersion}.tgz"
+
+    $embeddedPythonDir = "$BuildDir\embedded-python"
+    Download-File $EmbeddedPythonUrl "${embeddedPythonDir}.zip"
+    New-Item -Type Directory -Path $embeddedPythonDir
+    Expand-Archive "${embeddedPythonDir}.zip" $embeddedPythonDir
+    Remove-Item -Force "${embeddedPythonDir}.zip"
+
+    $sourcePythonDir = "$BuildDir\source-python"
+    Download-File "${SourcePythonUrl}" "${sourcePythonDir}.tgz"
+    New-Item -Type Directory -Path $sourcePythonDir
+    Expand-Archive "${sourcePythonDir}.tgz" $sourcePythonDir
+    Remove-Item -Force "${sourcePythonDir}.tgz"
+    New-Item -Type Directory -Path "$sourcePythonDir\src"
+    Expand-Archive "${sourcePythonDir}\source-python.tar" "$sourcePythonDir\src"
+    Remove-Item -Force -Recurse "${sourcePythonDir}\source-python.tar"
+    $sourcePythonDir = "${sourcePythonDir}\src\Python-3.7.7"
+
+    Remove-Item -Force "${embeddedPythonDir}\python37._pth"
+
+    New-Item -Type Directory -Path "${embeddedPythonDir}\Lib"
+    Expand-Archive "${embeddedPythonDir}\python37.zip" "${embeddedPythonDir}\Lib"
+    Remove-Item -Force "${embeddedPythonDir}\python37.zip"
+
+    Copy-Item -Recurse -Force "${sourcePythonDir}\Include" "${embeddedPythonDir}\"
+    Copy-Item -Recurse -Force "${sourcePythonDir}\PC\pyconfig.h" "${embeddedPythonDir}\Include\"
+
+    New-Item -Type Directory -Path "${embeddedPythonDir}\libs\"
+    # TODO: Needs to be replaced with the creation of lib from dll
+    Download-File "https://github.com/LuxCoreRender/WindowsCompileDeps/raw/master/x64/Release/lib/python37.lib" `
+        "${embeddedPythonDir}\libs\python37.lib"
+
+    $env:path = "${embeddedPythonDir};${embeddedPythonDir}\scripts;" + $env:path
+
+    Install-SetuptoolsFromSource $SetuptoolsUrl
+    # Comtypes cannot be installed as a requirement with pip install no_binary
+    Install-ComtypesFromSource $ComtypesUrl
+}
 ### Main ###
 
 try {
@@ -238,10 +345,17 @@ try {
     Set-VCVars -Version "14.0"
 
     # Setup pip upper requirements
-    Setup-PythonPip
+    # Setup-PythonPip
+
+    if ($EmbeddedPythonVersion) {
+        Setup-EmbeddedPythonEnvironment $EmbeddedPythonVersion
+    }
 
     # Install PyWin32 from source
-    Install-PyWin32FromSource $PyWin32RepoUrl
+    # Install-PyWin32FromSource $PyWin32RepoUrl
+    # TODO. Comment the following line and uncomment the line before. Keep this line for faster script testing (it takes more than 10 minutes to build the pywin32).
+    python -m pip install pywin32
+
 
     # PyMI setup can be skipped once the upstream version is published on pypi
     Install-PyMI $PyMiRepoUrl

From 94c20bd8f487ee2757b4068f90cfe01ae78dcf9b Mon Sep 17 00:00:00 2001
From: Adrian Vladu <avladu@cloudbasesolutions.com>
Date: Thu, 9 Apr 2020 20:49:34 +0300
Subject: [PATCH 04/25] added easy install wheel and pip from sources

Change-Id: I13c49a427139df06f216237266a42d19a26c4c96
---
 scripts/build_install_from_sources.ps1 | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scripts/build_install_from_sources.ps1 b/scripts/build_install_from_sources.ps1
index 15a06a51..d1261193 100644
--- a/scripts/build_install_from_sources.ps1
+++ b/scripts/build_install_from_sources.ps1
@@ -18,7 +18,7 @@ $PIP_BUILD_NO_BINARIES_ARGS = "--no-binary :all:"
 function Run-CmdWithRetry {
     param(
         $command,
-        [int]$maxRetryCount=1,
+        [int]$maxRetryCount=3,
         [int]$retryInterval=1
     )
 
@@ -276,8 +276,8 @@ function Install-SetuptoolsFromSource {
             Push-Location (Join-Path $BuildDir $SourcePath)
             Run-Command -Cmd "python" -Arguments @("-W ignore", ".\bootstrap.py")
             Run-Command -Cmd "python" -Arguments @("-W ignore", "setup.py", "install")
-            Run-Command -Cmd "python" -Arguments @("-W ignore", "-m", "easy_install", "pip")
-            Run-Command -Cmd "python" -Arguments @("-W ignore", "-m", "easy_install", "wheel")
+            Run-Command -Cmd "python" -Arguments @("-W ignore", "-m", "easy_install", "https://github.com/pypa/pip/archive/20.0.2.tar.gz")
+            Run-Command -Cmd "python" -Arguments @("-W ignore", "-m", "easy_install", "https://github.com/pypa/wheel/archive/0.34.2.tar.gz")
         } finally {
             Pop-Location
         }
@@ -352,9 +352,9 @@ try {
     }
 
     # Install PyWin32 from source
-    # Install-PyWin32FromSource $PyWin32RepoUrl
+    Install-PyWin32FromSource $PyWin32RepoUrl
     # TODO. Comment the following line and uncomment the line before. Keep this line for faster script testing (it takes more than 10 minutes to build the pywin32).
-    python -m pip install pywin32
+    # python -m pip install pywin32
 
 
     # PyMI setup can be skipped once the upstream version is published on pypi

From f52880557b73f7bdf1f9c70fe17a2d75ed415913 Mon Sep 17 00:00:00 2001
From: Adrian Vladu <avladu@cloudbasesolutions.com>
Date: Thu, 9 Apr 2020 20:53:03 +0300
Subject: [PATCH 05/25] Addde pip / wheel params for sources

Change-Id: Ic979742ae1df4cdb3a0cc4c3b474de0a40d175ee
---
 scripts/build_install_from_sources.ps1 | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scripts/build_install_from_sources.ps1 b/scripts/build_install_from_sources.ps1
index d1261193..1745caba 100644
--- a/scripts/build_install_from_sources.ps1
+++ b/scripts/build_install_from_sources.ps1
@@ -6,6 +6,8 @@ param(
     [string]$EmbeddedPythonVersion="3.7.7",
     [string]$ComtypesUrl="https://github.com/enthought/comtypes",
     [string]$SetuptoolsUrl="https://github.com/pypa/setuptools",
+    [string]$PipSourceUrl="https://github.com/pypa/pip/archive/20.0.2.tar.gz",
+    [string]$WheelSourceUrl="https://github.com/pypa/wheel/archive/0.34.2.tar.gz",
     [string]$BuildDir=""
 )
 
@@ -276,8 +278,8 @@ function Install-SetuptoolsFromSource {
             Push-Location (Join-Path $BuildDir $SourcePath)
             Run-Command -Cmd "python" -Arguments @("-W ignore", ".\bootstrap.py")
             Run-Command -Cmd "python" -Arguments @("-W ignore", "setup.py", "install")
-            Run-Command -Cmd "python" -Arguments @("-W ignore", "-m", "easy_install", "https://github.com/pypa/pip/archive/20.0.2.tar.gz")
-            Run-Command -Cmd "python" -Arguments @("-W ignore", "-m", "easy_install", "https://github.com/pypa/wheel/archive/0.34.2.tar.gz")
+            Run-Command -Cmd "python" -Arguments @("-W ignore", "-m", "easy_install", $PipSourceUrl)
+            Run-Command -Cmd "python" -Arguments @("-W ignore", "-m", "easy_install", $WheelSourceUrl)
         } finally {
             Pop-Location
         }

From 2d528fa6bbcf89ec8420bfaae1e1074078ee8be9 Mon Sep 17 00:00:00 2001
From: Adrian Vladu <avladu@cloudbasesolutions.com>
Date: Thu, 9 Apr 2020 21:00:26 +0300
Subject: [PATCH 06/25] Parameterize python version header

Change-Id: I88f24a8298565d07eb08cc4c4726d9434f5c18c7
---
 scripts/build_install_from_sources.ps1 | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/scripts/build_install_from_sources.ps1 b/scripts/build_install_from_sources.ps1
index 1745caba..b2d4078b 100644
--- a/scripts/build_install_from_sources.ps1
+++ b/scripts/build_install_from_sources.ps1
@@ -291,6 +291,7 @@ function Setup-EmbeddedPythonEnvironment {
 
     $EmbeddedPythonUrl = "https://www.python.org/ftp/python/${EmbeddedPythonVersion}/python-${EmbeddedPythonVersion}-embed-amd64.zip"
     $SourcePythonUrl = "https://www.python.org/ftp/python/${EmbeddedPythonVersion}/Python-${EmbeddedPythonVersion}.tgz"
+    $pythonVersionHeader = "python37"
 
     $embeddedPythonDir = "$BuildDir\embedded-python"
     Download-File $EmbeddedPythonUrl "${embeddedPythonDir}.zip"
@@ -306,21 +307,21 @@ function Setup-EmbeddedPythonEnvironment {
     New-Item -Type Directory -Path "$sourcePythonDir\src"
     Expand-Archive "${sourcePythonDir}\source-python.tar" "$sourcePythonDir\src"
     Remove-Item -Force -Recurse "${sourcePythonDir}\source-python.tar"
-    $sourcePythonDir = "${sourcePythonDir}\src\Python-3.7.7"
+    $sourcePythonDir = "${sourcePythonDir}\src\Python-${EmbeddedPythonVersion}"
 
-    Remove-Item -Force "${embeddedPythonDir}\python37._pth"
+    Remove-Item -Force "${embeddedPythonDir}\${pythonVersionHeader}._pth"
 
     New-Item -Type Directory -Path "${embeddedPythonDir}\Lib"
-    Expand-Archive "${embeddedPythonDir}\python37.zip" "${embeddedPythonDir}\Lib"
-    Remove-Item -Force "${embeddedPythonDir}\python37.zip"
+    Expand-Archive "${embeddedPythonDir}\${pythonVersionHeader}.zip" "${embeddedPythonDir}\Lib"
+    Remove-Item -Force "${embeddedPythonDir}\${pythonVersionHeader}.zip"
 
     Copy-Item -Recurse -Force "${sourcePythonDir}\Include" "${embeddedPythonDir}\"
     Copy-Item -Recurse -Force "${sourcePythonDir}\PC\pyconfig.h" "${embeddedPythonDir}\Include\"
 
     New-Item -Type Directory -Path "${embeddedPythonDir}\libs\"
     # TODO: Needs to be replaced with the creation of lib from dll
-    Download-File "https://github.com/LuxCoreRender/WindowsCompileDeps/raw/master/x64/Release/lib/python37.lib" `
-        "${embeddedPythonDir}\libs\python37.lib"
+    Download-File "https://github.com/LuxCoreRender/WindowsCompileDeps/raw/master/x64/Release/lib/${pythonVersionHeader}.lib" `
+        "${embeddedPythonDir}\libs\${pythonVersionHeader}.lib"
 
     $env:path = "${embeddedPythonDir};${embeddedPythonDir}\scripts;" + $env:path
 

From 9c462c04528fcdb472de992ccf024e2e6f04a2eb Mon Sep 17 00:00:00 2001
From: Adrian Vladu <avladu@cloudbasesolutions.com>
Date: Thu, 9 Apr 2020 21:07:04 +0300
Subject: [PATCH 07/25] Update the bdist_wininst.py from the embedded version
 with the full one

Change-Id: I399e91d0e10b4893a0ce9ec7ff6b07db93e5a1e1
---
 scripts/build_install_from_sources.ps1 | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/scripts/build_install_from_sources.ps1 b/scripts/build_install_from_sources.ps1
index b2d4078b..d235a2a3 100644
--- a/scripts/build_install_from_sources.ps1
+++ b/scripts/build_install_from_sources.ps1
@@ -326,8 +326,11 @@ function Setup-EmbeddedPythonEnvironment {
     $env:path = "${embeddedPythonDir};${embeddedPythonDir}\scripts;" + $env:path
 
     Install-SetuptoolsFromSource $SetuptoolsUrl
-    # Comtypes cannot be installed as a requirement with pip install no_binary
-    Install-ComtypesFromSource $ComtypesUrl
+    # Comtypes cannot be installed as a requirement with pip install no_binary if this bdist_winst is not replaced with the full Python version
+    $bdistWininstFile = "${embeddedPythonDir}\Lib\distutils\command\bdist_wininst.py"
+    Download-File "https://raw.githubusercontent.com/python/cpython/v${EmbeddedPythonVersion}/Lib/distutils/command/bdist_wininst.py" `
+        $bdistWininstFile
+    Remove-Item -Force "${bdistWininstFile}c"
 }
 ### Main ###
 

From 9bd40000dbc6f8d99a7ea230f4ad0aa7060decd0 Mon Sep 17 00:00:00 2001
From: Adrian Vladu <avladu@cloudbasesolutions.com>
Date: Thu, 9 Apr 2020 21:19:41 +0300
Subject: [PATCH 08/25] Removed hardcoded py prefix

Change-Id: I084da34a607f92416cce4b25c3ae822141c5a9ad
---
 scripts/build_install_from_sources.ps1 | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scripts/build_install_from_sources.ps1 b/scripts/build_install_from_sources.ps1
index d235a2a3..068ecb79 100644
--- a/scripts/build_install_from_sources.ps1
+++ b/scripts/build_install_from_sources.ps1
@@ -4,7 +4,6 @@ param(
     [string]$PyWin32RepoUrl="https://github.com/mhammond/pywin32",
     [string]$PyMiRepoUrl="https://github.com/cloudbase/PyMI",
     [string]$EmbeddedPythonVersion="3.7.7",
-    [string]$ComtypesUrl="https://github.com/enthought/comtypes",
     [string]$SetuptoolsUrl="https://github.com/pypa/setuptools",
     [string]$PipSourceUrl="https://github.com/pypa/pip/archive/20.0.2.tar.gz",
     [string]$WheelSourceUrl="https://github.com/pypa/wheel/archive/0.34.2.tar.gz",
@@ -291,7 +290,7 @@ function Setup-EmbeddedPythonEnvironment {
 
     $EmbeddedPythonUrl = "https://www.python.org/ftp/python/${EmbeddedPythonVersion}/python-${EmbeddedPythonVersion}-embed-amd64.zip"
     $SourcePythonUrl = "https://www.python.org/ftp/python/${EmbeddedPythonVersion}/Python-${EmbeddedPythonVersion}.tgz"
-    $pythonVersionHeader = "python37"
+    $pythonVersionHeader = "python" + (($EmbeddedPythonVersion.split(".") | Select-Object -First 2) -Join  "")
 
     $embeddedPythonDir = "$BuildDir\embedded-python"
     Download-File $EmbeddedPythonUrl "${embeddedPythonDir}.zip"

From 9790f7c8727fe0237d9ac4e893559f34029eaf78 Mon Sep 17 00:00:00 2001
From: Adrian Vladu <avladu@cloudbasesolutions.com>
Date: Thu, 9 Apr 2020 22:10:00 +0300
Subject: [PATCH 09/25] Build dll lib from the existend python dll

Change-Id: I41f4d636252cbd31bba27b9c870e4b06398bc297
---
 scripts/build_install_from_sources.ps1 | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/scripts/build_install_from_sources.ps1 b/scripts/build_install_from_sources.ps1
index 068ecb79..a649c627 100644
--- a/scripts/build_install_from_sources.ps1
+++ b/scripts/build_install_from_sources.ps1
@@ -318,9 +318,28 @@ function Setup-EmbeddedPythonEnvironment {
     Copy-Item -Recurse -Force "${sourcePythonDir}\PC\pyconfig.h" "${embeddedPythonDir}\Include\"
 
     New-Item -Type Directory -Path "${embeddedPythonDir}\libs\"
-    # TODO: Needs to be replaced with the creation of lib from dll
-    Download-File "https://github.com/LuxCoreRender/WindowsCompileDeps/raw/master/x64/Release/lib/${pythonVersionHeader}.lib" `
-        "${embeddedPythonDir}\libs\${pythonVersionHeader}.lib"
+    $pythonLibPath = "${embeddedPythonDir}\libs\${pythonVersionHeader}.lib"
+    $pythonLibDefRawPath = "${embeddedPythonDir}\libs\${pythonVersionHeader}.raw.def"
+    $pythonLibDefPath = "${embeddedPythonDir}\libs\${pythonVersionHeader}.def"
+    $pythonDllPath = "${embeddedPythonDir}\${pythonVersionHeader}.dll"
+    dumpbin.exe /exports "${pythonDllPath}" > "${pythonLibDefRawPath}"
+    if ($LastExitCode) {
+        throw "Failed to dump symbols"
+    }
+    Write-Output "EXPORTS" > "${pythonLibDefPath}"
+    Get-Content "${pythonLibDefRawPath}" | ForEach-Object {
+        $splitObject = $_.split(" ")
+        [array]::reverse($splitObject)
+        if ($splitObject[0] -clike "*Py*") {
+            Write-Output $splitObject[0] >> "${pythonLibDefPath}"
+        }
+    }
+    lib.exe /def:"${pythonLibDefPath}" /out:"${pythonLibPath}" /machine:x64
+    if ($LastExitCode) {
+        throw "Failed to create symbol map"
+    }
+    Remove-Item -Force $pythonLibDefRawPath
+    Remove-Item -Force $pythonLibDefPath
 
     $env:path = "${embeddedPythonDir};${embeddedPythonDir}\scripts;" + $env:path
 

From 3df14c6298193ffb38d3901d8370e62a1a413cd0 Mon Sep 17 00:00:00 2001
From: Adrian Vladu <avladu@cloudbasesolutions.com>
Date: Thu, 9 Apr 2020 22:17:31 +0300
Subject: [PATCH 10/25] fix some comments

Change-Id: Ie74e47faa23656fedbfb28f811a418eabfa06a83
---
 scripts/build_install_from_sources.ps1 | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scripts/build_install_from_sources.ps1 b/scripts/build_install_from_sources.ps1
index a649c627..3884850a 100644
--- a/scripts/build_install_from_sources.ps1
+++ b/scripts/build_install_from_sources.ps1
@@ -344,7 +344,9 @@ function Setup-EmbeddedPythonEnvironment {
     $env:path = "${embeddedPythonDir};${embeddedPythonDir}\scripts;" + $env:path
 
     Install-SetuptoolsFromSource $SetuptoolsUrl
-    # Comtypes cannot be installed as a requirement with pip install no_binary if this bdist_winst is not replaced with the full Python version
+
+    # Embedded Python has bdist_wininst.pyc reimplemented to throw an error if any package using this feature
+    # is to be installed. Pywin32 and comtypes packages cannot be installed with pip or by running setup.py install.
     $bdistWininstFile = "${embeddedPythonDir}\Lib\distutils\command\bdist_wininst.py"
     Download-File "https://raw.githubusercontent.com/python/cpython/v${EmbeddedPythonVersion}/Lib/distutils/command/bdist_wininst.py" `
         $bdistWininstFile

From 22ab71de10eb36466b2295998d42b302ec767f34 Mon Sep 17 00:00:00 2001
From: Adrian Vladu <avladu@cloudbasesolutions.com>
Date: Thu, 9 Apr 2020 22:23:29 +0300
Subject: [PATCH 11/25] Uncomment Setup-PythonPip

Change-Id: I3bd7ff072c0d5fa0346f32bfc0bd1661e45fa286
---
 scripts/build_install_from_sources.ps1 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/build_install_from_sources.ps1 b/scripts/build_install_from_sources.ps1
index 3884850a..9e18a7e1 100644
--- a/scripts/build_install_from_sources.ps1
+++ b/scripts/build_install_from_sources.ps1
@@ -371,7 +371,7 @@ try {
     Set-VCVars -Version "14.0"
 
     # Setup pip upper requirements
-    # Setup-PythonPip
+    Setup-PythonPip
 
     if ($EmbeddedPythonVersion) {
         Setup-EmbeddedPythonEnvironment $EmbeddedPythonVersion

From 93bed4b180cfcb7f78a0ba4273f28664494d237b Mon Sep 17 00:00:00 2001
From: Adrian Vladu <avladu@cloudbasesolutions.com>
Date: Thu, 9 Apr 2020 22:36:51 +0300
Subject: [PATCH 12/25] Fixed some new lines and updated readme

Change-Id: I30354f3e58a0adbc3ee0fdff72f2cc6c0830eacf
---
 scripts/Readme.md                      | 5 +++++
 scripts/build_install_from_sources.ps1 | 3 ++-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/scripts/Readme.md b/scripts/Readme.md
index 3d4165ce..4c71e770 100644
--- a/scripts/Readme.md
+++ b/scripts/Readme.md
@@ -25,6 +25,10 @@
     -CloudbaseInitRepoUrl "https://github.com/cloudbase/cloudbase-init" `
     -PyWin32RepoUrl "https://github.com/mhammond/pywin32" `
     -PyMiRepoUrl "https://github.com/cloudbase/PyMI" `
+    -EmbeddedPythonVersion "3.7.7" `
+    -SetuptoolsUrl "https://github.com/pypa/setuptools" `
+    -PipSourceUrl "https://github.com/pypa/pip/archive/20.0.2.tar.gz" `
+    -WheelSourceUrl "https://github.com/pypa/wheel/archive/0.34.2.tar.gz" `
     -BuildDir "build"
 
 ```
@@ -33,6 +37,7 @@
 #### Workflow of the script:
    - Download and set pip upper requirements from OpenStack
    - Create / clean temporary build directory
+   - If EmbeddedPythonVersion is set, download the Python embedded version, prepare it and add it to path
    - Build and install PyWin32 from sources
    - Build and install PyMI from sources
    - Build, install and create Cloudbase-Init binary from sources
diff --git a/scripts/build_install_from_sources.ps1 b/scripts/build_install_from_sources.ps1
index 9e18a7e1..9e282a5a 100644
--- a/scripts/build_install_from_sources.ps1
+++ b/scripts/build_install_from_sources.ps1
@@ -352,6 +352,8 @@ function Setup-EmbeddedPythonEnvironment {
         $bdistWininstFile
     Remove-Item -Force "${bdistWininstFile}c"
 }
+
+
 ### Main ###
 
 try {
@@ -382,7 +384,6 @@ try {
     # TODO. Comment the following line and uncomment the line before. Keep this line for faster script testing (it takes more than 10 minutes to build the pywin32).
     # python -m pip install pywin32
 
-
     # PyMI setup can be skipped once the upstream version is published on pypi
     Install-PyMI $PyMiRepoUrl
 

From e13e13183b28204ddbdef229c753bfe6f30775d5 Mon Sep 17 00:00:00 2001
From: Adrian Vladu <avladu@cloudbasesolutions.com>
Date: Fri, 10 Apr 2020 21:10:29 +0300
Subject: [PATCH 13/25] Added support to build Python from sources

Change-Id: I34a3c605443c64107fc4199abf63145a7654cf3a
---
 scripts/build_install_from_sources.ps1 | 48 +++++++++++++++++++++++---
 1 file changed, 44 insertions(+), 4 deletions(-)

diff --git a/scripts/build_install_from_sources.ps1 b/scripts/build_install_from_sources.ps1
index 9e282a5a..a6d30ffc 100644
--- a/scripts/build_install_from_sources.ps1
+++ b/scripts/build_install_from_sources.ps1
@@ -3,7 +3,8 @@ param(
     [string]$CloudbaseInitRepoUrl="https://github.com/cloudbase/cloudbase-init",
     [string]$PyWin32RepoUrl="https://github.com/mhammond/pywin32",
     [string]$PyMiRepoUrl="https://github.com/cloudbase/PyMI",
-    [string]$EmbeddedPythonVersion="3.7.7",
+    [string]$FromSourcePythonVersion="v3.7.7",
+    [string]$EmbeddedPythonVersion="",
     [string]$SetuptoolsUrl="https://github.com/pypa/setuptools",
     [string]$PipSourceUrl="https://github.com/pypa/pip/archive/20.0.2.tar.gz",
     [string]$WheelSourceUrl="https://github.com/pypa/wheel/archive/0.34.2.tar.gz",
@@ -101,7 +102,8 @@ function Run-Command {
 function Clone-Repo {
     param(
         $Url,
-        $Destination
+        $Destination,
+        $Branch="master"
     )
 
     Write-Host "Cloning ${Url} to ${Destination}"
@@ -109,9 +111,9 @@ function Clone-Repo {
     Run-CmdWithRetry {
         try {
             Push-Location $BuildDir
-            git clone $Url $Destination
+            git clone --single-branch -b $Branch $Url $Destination
             if ($LastExitCode) {
-                throw "git clone ${Url} ${Destination} failed"
+                throw "git clone --single-branch -b $Url $Branch $Destination failed"
             }
         } finally {
             Pop-Location
@@ -353,6 +355,40 @@ function Setup-EmbeddedPythonEnvironment {
     Remove-Item -Force "${bdistWininstFile}c"
 }
 
+function Setup-FromSourcePythonEnvironment {
+    param($FromSourcePythonVersion)
+
+    $sourceFolder = "python-source"
+    $pythonSourceFolderPath = "$BuildDir\$sourceFolder"
+    $pythonTempBuildDir = "$pythonSourceFolderPath\PCbuild\amd64"
+    $pythonSourceRepo = "https://github.com/python/cpython"
+    $pythonBuildDir = "$BuildDir\python"
+
+    Clone-Repo -Url $pythonSourceRepo -Destination $sourceFolder -Branch $FromSourcePythonVersion
+    Push-Location $pythonSourceFolderPath
+    try {
+        cmd.exe /c PCbuild\build.bat -p x64 --no-tkinter -e
+        if ($LastExitCode) {
+            throw "Failed to build Python from source"
+        }
+    } finally {
+        Pop-Location
+    }
+
+    Move-Item $pythonTempBuildDir $pythonBuildDir
+    Copy-Item -Recurse "$pythonSourceFolderPath\Include" "$pythonBuildDir"
+    Copy-Item "$pythonSourceFolderPath\PC\pyconfig.h" "$pythonBuildDir\Include\"
+
+    Copy-Item -Recurse "$pythonSourceFolderPath\Lib" "$pythonBuildDir"
+
+    New-Item -Type Directory -Path "${pythonBuildDir}\libs\"
+    Copy-Item "$pythonBuildDir\python*.lib" "${pythonBuildDir}\libs\"
+
+    $env:path = "${pythonBuildDir};${pythonBuildDir}\scripts;" + $env:path
+
+    Install-SetuptoolsFromSource $SetuptoolsUrl
+}
+
 
 ### Main ###
 
@@ -379,6 +415,10 @@ try {
         Setup-EmbeddedPythonEnvironment $EmbeddedPythonVersion
     }
 
+    if ($FromSourcePythonVersion) {
+        Setup-FromSourcePythonEnvironment $FromSourcePythonVersion
+    }
+
     # Install PyWin32 from source
     Install-PyWin32FromSource $PyWin32RepoUrl
     # TODO. Comment the following line and uncomment the line before. Keep this line for faster script testing (it takes more than 10 minutes to build the pywin32).

From dc0b987d47540d81fefc52c48670ec1cda5962db Mon Sep 17 00:00:00 2001
From: Adrian Vladu <avladu@cloudbasesolutions.com>
Date: Mon, 13 Apr 2020 16:26:04 +0300
Subject: [PATCH 14/25] Added better parameters and cleanup noop

Change-Id: I005f3a664f11a7a47e083151a8ce6cb72fbb9bb6
---
 scripts/Readme.md                      |  21 +++--
 scripts/build_install_from_sources.ps1 | 103 +++++++++++++++----------
 2 files changed, 77 insertions(+), 47 deletions(-)

diff --git a/scripts/Readme.md b/scripts/Readme.md
index 4c71e770..45ecb6a8 100644
--- a/scripts/Readme.md
+++ b/scripts/Readme.md
@@ -2,7 +2,7 @@
 
 #### Clean Windows Server 2016 /2019 or Windows 10 install with latest updates
 
-#### Python 3.7 installed and added to path (with pip installed).
+#### Python installed and added to path (with pip installed).
     Download link: https://www.python.org/ftp/python/3.7.7/python-3.7.7-amd64.exe
 
 #### Visual Studio 2015 Community installed, with the following components:
@@ -13,9 +13,15 @@
     If you prefer to not build pywin32, Visual Studio 2017 / 2019 can be used too.
     Reboot after VS installation is complete.
 
+### To build Python from source, Visual Studio 2017 Community installed with the following components:
+    - VC++ 2017 v141
+    - Windows 10 SDK (10.0.14393.0). This version supports Python 3.8 build too.
+
 #### All the Python packages will be built and installed using pip flags: "--no-binary :all:"
 
-## The full build of cloudbase-init (and dependencies) from sources takes around 10 minutes.
+## The full build of cloudbase-init (and dependencies) using Embedded Python takes around 10 minutes.
+
+## The full build of cloudbase-init (and dependencies) using Python from source takes around 20 minutes.
 
 #### How to run:
 
@@ -25,10 +31,12 @@
     -CloudbaseInitRepoUrl "https://github.com/cloudbase/cloudbase-init" `
     -PyWin32RepoUrl "https://github.com/mhammond/pywin32" `
     -PyMiRepoUrl "https://github.com/cloudbase/PyMI" `
-    -EmbeddedPythonVersion "3.7.7" `
+    -PythonOrigin "AlreadyInstalled" `
+    -PythonVersion "v3.7.7" `
     -SetuptoolsUrl "https://github.com/pypa/setuptools" `
     -PipSourceUrl "https://github.com/pypa/pip/archive/20.0.2.tar.gz" `
-    -WheelSourceUrl "https://github.com/pypa/wheel/archive/0.34.2.tar.gz" `
+    -WheelSourceUrl "https://github.com/pypa/wheel/archive/0.34.2.tar.gz" `,
+    -CleanBuildArtifacts:$false,
     -BuildDir "build"
 
 ```
@@ -37,7 +45,10 @@
 #### Workflow of the script:
    - Download and set pip upper requirements from OpenStack
    - Create / clean temporary build directory
-   - If EmbeddedPythonVersion is set, download the Python embedded version, prepare it and add it to path
+   - If PythonOrigin="AlreadyInstalled" is set, do nothing. Python should be already installed and added to path.
+   - If PythonOrigin="FromSource" is set, download the Python source from GitHub, build, prepare it and add it to path. Setuptools, pip and wheel will be built from source and installed.
+   - If PythonOrigin="Embedded" is set, download the Python embedded, prepare it and add it to path. Setuptools, pip and wheel will be built from source and installed.
    - Build and install PyWin32 from sources
    - Build and install PyMI from sources
    - Build, install and create Cloudbase-Init binary from sources
+   - If CleanBuildArtifacts is set adn PythonOrigin is Embedded or FromSource, cleanup the .pdb, .pyc, header files.
diff --git a/scripts/build_install_from_sources.ps1 b/scripts/build_install_from_sources.ps1
index a6d30ffc..b2e4361b 100644
--- a/scripts/build_install_from_sources.ps1
+++ b/scripts/build_install_from_sources.ps1
@@ -1,13 +1,26 @@
 #ps1
 param(
+    [parameter(Mandatory=$true)]
     [string]$CloudbaseInitRepoUrl="https://github.com/cloudbase/cloudbase-init",
+    [parameter(Mandatory=$true)]
     [string]$PyWin32RepoUrl="https://github.com/mhammond/pywin32",
+    [parameter(Mandatory=$true)]
     [string]$PyMiRepoUrl="https://github.com/cloudbase/PyMI",
-    [string]$FromSourcePythonVersion="v3.7.7",
-    [string]$EmbeddedPythonVersion="",
-    [string]$SetuptoolsUrl="https://github.com/pypa/setuptools",
+    [parameter(Mandatory=$true)]
+    [ValidateSet("AlreadyInstalled", "Embedded", "FromSource")]
+    [string]$PythonOrigin="AlreadyInstalled",
+    # Specifies which version to use in case Embedded or FromSource is used
+    # In Embedded case: "https://www.python.org/ftp/python/$PythonVersion" should exist
+    # In FromSource case, the GitHub https://github.com/python/cpython branch should exist (tags also work as branches)
+    [string]$PythonVersion="v3.7.7",
+    # Specifies which setuptools git source to use in case Embedded or FromSource is used
+    [string]$SetuptoolsGitUrl="https://github.com/pypa/setuptools",
+    # Specifies which pip source in ttar.gz format to use in case Embedded or FromSource is used
     [string]$PipSourceUrl="https://github.com/pypa/pip/archive/20.0.2.tar.gz",
+    # Specifies which wheel source in ttar.gz format to use in case Embedded or FromSource is used
     [string]$WheelSourceUrl="https://github.com/pypa/wheel/archive/0.34.2.tar.gz",
+    # If the PythonOrigin is set to Embedded or FromSource, clean the Python folder (remove .pdb, .pyc, header files)
+    [switch]$CleanBuildArtifacts,
     [string]$BuildDir=""
 )
 
@@ -213,15 +226,6 @@ function Setup-PythonPip {
     $env:PIP_CONSTRAINT = ""
     $env:PIP_NO_BINARY = ""
 
-    # Update pip. Not needed for latest version of Python 3.7.
-    # Download-File "https://bootstrap.pypa.io/get-pip.py" "${BuildDir}/get-pip.py"
-    # Run-CmdWithRetry {
-    #    python "${BuildDir}/get-pip.py"
-    #    if ($LastExitCode) {
-    #        throw "Failed to run 'python get-pip.py'"
-    #    }
-    #}
-
     # Cloudbase-Init Python requirements should respect the OpenStack upper constraints
     Clone-Repo "https://github.com/openstack/requirements" "requirements"
     $constraintsFilePath = Join-Path $BuildDir "requirements/upper-constraints.txt"
@@ -294,11 +298,10 @@ function Setup-EmbeddedPythonEnvironment {
     $SourcePythonUrl = "https://www.python.org/ftp/python/${EmbeddedPythonVersion}/Python-${EmbeddedPythonVersion}.tgz"
     $pythonVersionHeader = "python" + (($EmbeddedPythonVersion.split(".") | Select-Object -First 2) -Join  "")
 
-    $embeddedPythonDir = "$BuildDir\embedded-python"
-    Download-File $EmbeddedPythonUrl "${embeddedPythonDir}.zip"
-    New-Item -Type Directory -Path $embeddedPythonDir
-    Expand-Archive "${embeddedPythonDir}.zip" $embeddedPythonDir
-    Remove-Item -Force "${embeddedPythonDir}.zip"
+    Download-File $EmbeddedPythonUrl "${PythonDir}.zip"
+    New-Item -Type Directory -Path $PythonDir
+    Expand-Archive "${PythonDir}.zip" $PythonDir
+    Remove-Item -Force "${PythonDir}.zip"
 
     $sourcePythonDir = "$BuildDir\source-python"
     Download-File "${SourcePythonUrl}" "${sourcePythonDir}.tgz"
@@ -310,20 +313,20 @@ function Setup-EmbeddedPythonEnvironment {
     Remove-Item -Force -Recurse "${sourcePythonDir}\source-python.tar"
     $sourcePythonDir = "${sourcePythonDir}\src\Python-${EmbeddedPythonVersion}"
 
-    Remove-Item -Force "${embeddedPythonDir}\${pythonVersionHeader}._pth"
+    Remove-Item -Force "${PythonDir}\${pythonVersionHeader}._pth"
 
-    New-Item -Type Directory -Path "${embeddedPythonDir}\Lib"
-    Expand-Archive "${embeddedPythonDir}\${pythonVersionHeader}.zip" "${embeddedPythonDir}\Lib"
-    Remove-Item -Force "${embeddedPythonDir}\${pythonVersionHeader}.zip"
+    New-Item -Type Directory -Path "${PythonDir}\Lib"
+    Expand-Archive "${PythonDir}\${pythonVersionHeader}.zip" "${PythonDir}\Lib"
+    Remove-Item -Force "${PythonDir}\${pythonVersionHeader}.zip"
 
-    Copy-Item -Recurse -Force "${sourcePythonDir}\Include" "${embeddedPythonDir}\"
-    Copy-Item -Recurse -Force "${sourcePythonDir}\PC\pyconfig.h" "${embeddedPythonDir}\Include\"
+    Copy-Item -Recurse -Force "${sourcePythonDir}\Include" "${PythonDir}\"
+    Copy-Item -Recurse -Force "${sourcePythonDir}\PC\pyconfig.h" "${PythonDir}\Include\"
 
-    New-Item -Type Directory -Path "${embeddedPythonDir}\libs\"
-    $pythonLibPath = "${embeddedPythonDir}\libs\${pythonVersionHeader}.lib"
-    $pythonLibDefRawPath = "${embeddedPythonDir}\libs\${pythonVersionHeader}.raw.def"
-    $pythonLibDefPath = "${embeddedPythonDir}\libs\${pythonVersionHeader}.def"
-    $pythonDllPath = "${embeddedPythonDir}\${pythonVersionHeader}.dll"
+    New-Item -Type Directory -Path "${PythonDir}\libs\"
+    $pythonLibPath = "${PythonDir}\libs\${pythonVersionHeader}.lib"
+    $pythonLibDefRawPath = "${PythonDir}\libs\${pythonVersionHeader}.raw.def"
+    $pythonLibDefPath = "${PythonDir}\libs\${pythonVersionHeader}.def"
+    $pythonDllPath = "${PythonDir}\${pythonVersionHeader}.dll"
     dumpbin.exe /exports "${pythonDllPath}" > "${pythonLibDefRawPath}"
     if ($LastExitCode) {
         throw "Failed to dump symbols"
@@ -345,7 +348,7 @@ function Setup-EmbeddedPythonEnvironment {
 
     $env:path = "${embeddedPythonDir};${embeddedPythonDir}\scripts;" + $env:path
 
-    Install-SetuptoolsFromSource $SetuptoolsUrl
+    Install-SetuptoolsFromSource $SetuptoolsGitUrl
 
     # Embedded Python has bdist_wininst.pyc reimplemented to throw an error if any package using this feature
     # is to be installed. Pywin32 and comtypes packages cannot be installed with pip or by running setup.py install.
@@ -362,7 +365,6 @@ function Setup-FromSourcePythonEnvironment {
     $pythonSourceFolderPath = "$BuildDir\$sourceFolder"
     $pythonTempBuildDir = "$pythonSourceFolderPath\PCbuild\amd64"
     $pythonSourceRepo = "https://github.com/python/cpython"
-    $pythonBuildDir = "$BuildDir\python"
 
     Clone-Repo -Url $pythonSourceRepo -Destination $sourceFolder -Branch $FromSourcePythonVersion
     Push-Location $pythonSourceFolderPath
@@ -375,18 +377,19 @@ function Setup-FromSourcePythonEnvironment {
         Pop-Location
     }
 
-    Move-Item $pythonTempBuildDir $pythonBuildDir
-    Copy-Item -Recurse "$pythonSourceFolderPath\Include" "$pythonBuildDir"
-    Copy-Item "$pythonSourceFolderPath\PC\pyconfig.h" "$pythonBuildDir\Include\"
+    Move-Item $pythonTempBuildDir $PythonDir
+    Copy-Item -Recurse "$pythonSourceFolderPath\Include" "$PythonDir"
+    Copy-Item "$pythonSourceFolderPath\PC\pyconfig.h" "$PythonDir\Include\"
 
-    Copy-Item -Recurse "$pythonSourceFolderPath\Lib" "$pythonBuildDir"
+    Copy-Item -Recurse "$pythonSourceFolderPath\Lib" "$PythonDir"
 
-    New-Item -Type Directory -Path "${pythonBuildDir}\libs\"
-    Copy-Item "$pythonBuildDir\python*.lib" "${pythonBuildDir}\libs\"
+    New-Item -Type Directory -Path "${PythonDir}\libs\"
+    Copy-Item "$PythonDir\python*.lib" "${PythonDir}\libs\"
 
-    $env:path = "${pythonBuildDir};${pythonBuildDir}\scripts;" + $env:path
+    Install-SetuptoolsFromSource $SetuptoolsGitUrl
+}
 
-    Install-SetuptoolsFromSource $SetuptoolsUrl
+function Clean-BuildArtifacts {
 }
 
 
@@ -401,7 +404,9 @@ try {
         $BuildDir = "build"
     }
     if (![System.IO.Path]::IsPathRooted($BuildDir)) {
+        # There are used as global variables and their value do not change
         $BuildDir = Join-Path $scriptPath $BuildDir
+        $PythonDir = Join-Path $BuildDir "python"
     }
     Prepare-BuildDir
 
@@ -411,12 +416,22 @@ try {
     # Setup pip upper requirements
     Setup-PythonPip
 
-    if ($EmbeddedPythonVersion) {
-        Setup-EmbeddedPythonEnvironment $EmbeddedPythonVersion
+    if ($PythonOrigin -eq "Embedded") {
+        if ($PythonVersion -and $SetuptoolsGitUrl -and $PipSourceUrl -and $WheelSourceUrl) {
+            Setup-EmbeddedPythonEnvironment $PythonVersion
+        } else {
+            throw "If PythonOrigin is set to Embedded, SetuptoolsGitUrl, PipSourceUrl and WheelSourceUrl must be set"
+        }
+        $env:path = "${PythonDir};${PythonDir}\scripts;" + $env:path
     }
 
-    if ($FromSourcePythonVersion) {
-        Setup-FromSourcePythonEnvironment $FromSourcePythonVersion
+    if ($PythonOrigin -eq "FromSource") {
+        if ($PythonVersion -and $SetuptoolsGitUrl -and $PipSourceUrl -and $WheelSourceUrl) {
+            Setup-FromSourcePythonEnvironment $PythonVersion
+        } else {
+            throw "If PythonOrigin is set to FromSource, SetuptoolsGitUrl, PipSourceUrl and WheelSourceUrl must be set"
+        }
+        $env:path = "${PythonDir};${PythonDir}\scripts;" + $env:path
     }
 
     # Install PyWin32 from source
@@ -428,6 +443,10 @@ try {
     Install-PyMI $PyMiRepoUrl
 
     Install-CloudbaseInit $CloudbaseInitRepoUrl
+
+    if ($CleanBuildArtifacts -and ($PythonOrigin -eq "Embedded" -or $PythonOrigin -eq "FromSource")) {
+        Clean-BuildArtifacts
+    }
 } finally {
     $endDate = Get-Date
     Write-Host "Cloudbase-Init build finished after $(($endDate - $StartDate).Minutes + 1) minutes."

From 1cb3a43b5987eed3f253c3563bd3e67a4934756f Mon Sep 17 00:00:00 2001
From: Adrian Vladu <avladu@cloudbasesolutions.com>
Date: Mon, 13 Apr 2020 16:28:00 +0300
Subject: [PATCH 15/25] Fix readme headers

Change-Id: I8a13671953c3c26c55fa780e705ae3d9b7463442
---
 scripts/Readme.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scripts/Readme.md b/scripts/Readme.md
index 45ecb6a8..aed6bdeb 100644
--- a/scripts/Readme.md
+++ b/scripts/Readme.md
@@ -2,7 +2,7 @@
 
 #### Clean Windows Server 2016 /2019 or Windows 10 install with latest updates
 
-#### Python installed and added to path (with pip installed).
+#### Python installed and added to path (with pip installed) if PythonOrigin is set to AlreadyInstalled.
     Download link: https://www.python.org/ftp/python/3.7.7/python-3.7.7-amd64.exe
 
 #### Visual Studio 2015 Community installed, with the following components:
@@ -13,15 +13,15 @@
     If you prefer to not build pywin32, Visual Studio 2017 / 2019 can be used too.
     Reboot after VS installation is complete.
 
-### To build Python from source, Visual Studio 2017 Community installed with the following components:
+#### To build Python from source, Visual Studio 2017 Community installed with the following components:
     - VC++ 2017 v141
     - Windows 10 SDK (10.0.14393.0). This version supports Python 3.8 build too.
 
 #### All the Python packages will be built and installed using pip flags: "--no-binary :all:"
 
-## The full build of cloudbase-init (and dependencies) using Embedded Python takes around 10 minutes.
+#### The full build of cloudbase-init (and dependencies) using Embedded Python takes around 10 minutes.
 
-## The full build of cloudbase-init (and dependencies) using Python from source takes around 20 minutes.
+#### The full build of cloudbase-init (and dependencies) using Python from source takes around 20 minutes.
 
 #### How to run:
 
@@ -51,4 +51,4 @@
    - Build and install PyWin32 from sources
    - Build and install PyMI from sources
    - Build, install and create Cloudbase-Init binary from sources
-   - If CleanBuildArtifacts is set adn PythonOrigin is Embedded or FromSource, cleanup the .pdb, .pyc, header files.
+   - If CleanBuildArtifacts is set and PythonOrigin is Embedded or FromSource, cleanup the .pdb, .pyc, header files.

From 7add7f0b62f7d593b7f72d5e46b445c8b61a6939 Mon Sep 17 00:00:00 2001
From: Adrian Vladu <avladu@cloudbasesolutions.com>
Date: Mon, 13 Apr 2020 16:48:34 +0300
Subject: [PATCH 16/25] Fixes after refactor

Change-Id: I6cbfa933ae692241e7cd25fadabd7faa83807dcc
---
 scripts/build_install_from_sources.ps1 | 33 +++++++-------------------
 1 file changed, 9 insertions(+), 24 deletions(-)

diff --git a/scripts/build_install_from_sources.ps1 b/scripts/build_install_from_sources.ps1
index b2e4361b..2ea857a5 100644
--- a/scripts/build_install_from_sources.ps1
+++ b/scripts/build_install_from_sources.ps1
@@ -1,12 +1,8 @@
 #ps1
 param(
-    [parameter(Mandatory=$true)]
     [string]$CloudbaseInitRepoUrl="https://github.com/cloudbase/cloudbase-init",
-    [parameter(Mandatory=$true)]
     [string]$PyWin32RepoUrl="https://github.com/mhammond/pywin32",
-    [parameter(Mandatory=$true)]
     [string]$PyMiRepoUrl="https://github.com/cloudbase/PyMI",
-    [parameter(Mandatory=$true)]
     [ValidateSet("AlreadyInstalled", "Embedded", "FromSource")]
     [string]$PythonOrigin="AlreadyInstalled",
     # Specifies which version to use in case Embedded or FromSource is used
@@ -346,10 +342,6 @@ function Setup-EmbeddedPythonEnvironment {
     Remove-Item -Force $pythonLibDefRawPath
     Remove-Item -Force $pythonLibDefPath
 
-    $env:path = "${embeddedPythonDir};${embeddedPythonDir}\scripts;" + $env:path
-
-    Install-SetuptoolsFromSource $SetuptoolsGitUrl
-
     # Embedded Python has bdist_wininst.pyc reimplemented to throw an error if any package using this feature
     # is to be installed. Pywin32 and comtypes packages cannot be installed with pip or by running setup.py install.
     $bdistWininstFile = "${embeddedPythonDir}\Lib\distutils\command\bdist_wininst.py"
@@ -385,8 +377,6 @@ function Setup-FromSourcePythonEnvironment {
 
     New-Item -Type Directory -Path "${PythonDir}\libs\"
     Copy-Item "$PythonDir\python*.lib" "${PythonDir}\libs\"
-
-    Install-SetuptoolsFromSource $SetuptoolsGitUrl
 }
 
 function Clean-BuildArtifacts {
@@ -416,35 +406,30 @@ try {
     # Setup pip upper requirements
     Setup-PythonPip
 
-    if ($PythonOrigin -eq "Embedded") {
-        if ($PythonVersion -and $SetuptoolsGitUrl -and $PipSourceUrl -and $WheelSourceUrl) {
+    if (@("Embedded", "FromSource") -contains $PythonOrigin) {
+        if (!($PythonVersion -and $SetuptoolsGitUrl -and $PipSourceUrl -and $WheelSourceUrl)) {
+            throw "If PythonOrigin is set to ${PythonOrigin}, SetuptoolsGitUrl, PipSourceUrl and WheelSourceUrl must be set"
+        }
+        if ($PythonOrigin -eq "Embedded") {
             Setup-EmbeddedPythonEnvironment $PythonVersion
         } else {
-            throw "If PythonOrigin is set to Embedded, SetuptoolsGitUrl, PipSourceUrl and WheelSourceUrl must be set"
-        }
-        $env:path = "${PythonDir};${PythonDir}\scripts;" + $env:path
-    }
-
-    if ($PythonOrigin -eq "FromSource") {
-        if ($PythonVersion -and $SetuptoolsGitUrl -and $PipSourceUrl -and $WheelSourceUrl) {
             Setup-FromSourcePythonEnvironment $PythonVersion
-        } else {
-            throw "If PythonOrigin is set to FromSource, SetuptoolsGitUrl, PipSourceUrl and WheelSourceUrl must be set"
         }
         $env:path = "${PythonDir};${PythonDir}\scripts;" + $env:path
+        Install-SetuptoolsFromSource $SetuptoolsGitUrl
     }
 
     # Install PyWin32 from source
-    Install-PyWin32FromSource $PyWin32RepoUrl
+    #Install-PyWin32FromSource $PyWin32RepoUrl
     # TODO. Comment the following line and uncomment the line before. Keep this line for faster script testing (it takes more than 10 minutes to build the pywin32).
-    # python -m pip install pywin32
+    python -m pip install pywin32
 
     # PyMI setup can be skipped once the upstream version is published on pypi
     Install-PyMI $PyMiRepoUrl
 
     Install-CloudbaseInit $CloudbaseInitRepoUrl
 
-    if ($CleanBuildArtifacts -and ($PythonOrigin -eq "Embedded" -or $PythonOrigin -eq "FromSource")) {
+    if ($CleanBuildArtifacts -and (@("Embedded", "FromSource") -contains $PythonOrigin)) {
         Clean-BuildArtifacts
     }
 } finally {

From 0cfdc7ddf8b837daa5f461274f814ced9d0a42cf Mon Sep 17 00:00:00 2001
From: Adrian Vladu <avladu@cloudbasesolutions.com>
Date: Mon, 13 Apr 2020 17:21:02 +0300
Subject: [PATCH 17/25] Added test config for empty metadata source

Change-Id: Ida2175a477fee7f1b4d6db7ceaa1bcd7edf85355
---
 scripts/build_install_from_sources.ps1   | 13 +++++++++----
 scripts/cloudbase-init-empty-source.conf |  7 +++++++
 2 files changed, 16 insertions(+), 4 deletions(-)
 create mode 100644 scripts/cloudbase-init-empty-source.conf

diff --git a/scripts/build_install_from_sources.ps1 b/scripts/build_install_from_sources.ps1
index 2ea857a5..be449cc3 100644
--- a/scripts/build_install_from_sources.ps1
+++ b/scripts/build_install_from_sources.ps1
@@ -344,10 +344,9 @@ function Setup-EmbeddedPythonEnvironment {
 
     # Embedded Python has bdist_wininst.pyc reimplemented to throw an error if any package using this feature
     # is to be installed. Pywin32 and comtypes packages cannot be installed with pip or by running setup.py install.
-    $bdistWininstFile = "${embeddedPythonDir}\Lib\distutils\command\bdist_wininst.py"
-    Download-File "https://raw.githubusercontent.com/python/cpython/v${EmbeddedPythonVersion}/Lib/distutils/command/bdist_wininst.py" `
-        $bdistWininstFile
-    Remove-Item -Force "${bdistWininstFile}c"
+    $bdistFile = "Lib\distutils\command\bdist_wininst.py"
+    Copy-Item -Force "${sourcePythonDir}\${bdistFile}" "${PythonDir}\${bdistFile}"
+    Remove-Item -Force "${PythonDir}\${bdistFile}c"
 }
 
 function Setup-FromSourcePythonEnvironment {
@@ -380,6 +379,12 @@ function Setup-FromSourcePythonEnvironment {
 }
 
 function Clean-BuildArtifacts {
+    # Remove the Include folder
+    Get-Item "${PythonDir\Include}"
+    # Remove all the .pdb files
+    Get-ChildItem -Recurse -Include "*.pdb" $PythonDir
+    # Remove all the .pyc files that do have a .py correspondent
+    Get-ChildItem -Recurse -Include "*.pyc" $PythonDir
 }
 
 
diff --git a/scripts/cloudbase-init-empty-source.conf b/scripts/cloudbase-init-empty-source.conf
new file mode 100644
index 00000000..62149fc4
--- /dev/null
+++ b/scripts/cloudbase-init-empty-source.conf
@@ -0,0 +1,7 @@
+[DEFAULT]
+debug = true
+logfile = 
+allow_reboot = false
+stop_service_on_exit = false
+metadata_services = cloudbaseinit.metadata.services.base.EmptyMetadataService
+plugins = cloudbaseinit.plugins.common.mtu.MTUPlugin,cloudbaseinit.plugins.windows.ntpclient.NTPClientPlugin,cloudbaseinit.plugins.windows.sanpolicy.SANPolicyPlugin,cloudbaseinit.plugins.windows.displayidletimeout.DisplayIdleTimeoutConfigPlugin,cloudbaseinit.plugins.windows.bootconfig.BootStatusPolicyPlugin,cloudbaseinit.plugins.common.sethostname.SetHostNamePlugin,cloudbaseinit.plugins.windows.extendvolumes.ExtendVolumesPlugin,cloudbaseinit.plugins.common.userdata.UserDataPlugin,cloudbaseinit.plugins.windows.winrmlistener.ConfigWinRMListenerPlugin,cloudbaseinit.plugins.common.localscripts.LocalScriptsPlugin,cloudbaseinit.plugins.common.trim.TrimConfigPlugin,

From 668a528ea19e51d4317558b75104a6ceeb2256b8 Mon Sep 17 00:00:00 2001
From: Adrian Vladu <avladu@cloudbasesolutions.com>
Date: Mon, 13 Apr 2020 17:52:04 +0300
Subject: [PATCH 18/25] Added proper cleanup for the build artifacts

Change-Id: Ie9005ed14e0301cc0431e43da78cda5a0efe724b
---
 scripts/build_install_from_sources.ps1 | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/scripts/build_install_from_sources.ps1 b/scripts/build_install_from_sources.ps1
index be449cc3..c5a88dd8 100644
--- a/scripts/build_install_from_sources.ps1
+++ b/scripts/build_install_from_sources.ps1
@@ -255,16 +255,19 @@ function Install-PyMI {
     Clone-Repo $Url $sourcePath
     Install-PythonRequirements -SourcePath $sourcePath -BuildWithoutBinaries
     Install-PythonPackage -SourcePath $sourcePath
+    Write-Host "PyMI installed successfully"
 }
 
 function Install-CloudbaseInit {
     param($Url)
 
+    Write-Host "Installing CloudbaseInit..."
     $sourcePath = "cloudbase-init"
 
     Clone-Repo $Url $sourcePath
     Install-PythonRequirements -SourcePath $sourcePath -BuildWithoutBinaries
     Install-PythonPackage -SourcePath $sourcePath
+    Write-Host "CloudbaseInit installed successfully"
 }
 
 function Install-SetuptoolsFromSource {
@@ -380,11 +383,13 @@ function Setup-FromSourcePythonEnvironment {
 
 function Clean-BuildArtifacts {
     # Remove the Include folder
-    Get-Item "${PythonDir\Include}"
+    Remove-Item -Force -Recurse "${PythonDir}\Include"
+    # Remove all the __pycache__ folders
+    Get-ChildItem -Recurse -Include "__pycache__" $PythonDir | Remove-Item -Force -Recurse
     # Remove all the .pdb files
-    Get-ChildItem -Recurse -Include "*.pdb" $PythonDir
-    # Remove all the .pyc files that do have a .py correspondent
-    Get-ChildItem -Recurse -Include "*.pyc" $PythonDir
+    Get-ChildItem -Recurse -Include "*.pdb" $PythonDir | Remove-Item -Force
+    # Remove lib and exp files
+    Get-ChildItem -Include @("*.lib", "*.exp") -Exclude "python*" "$PythonDir\*" | Remove-Item -Force
 }
 
 
@@ -425,9 +430,9 @@ try {
     }
 
     # Install PyWin32 from source
-    #Install-PyWin32FromSource $PyWin32RepoUrl
+    Install-PyWin32FromSource $PyWin32RepoUrl
     # TODO. Comment the following line and uncomment the line before. Keep this line for faster script testing (it takes more than 10 minutes to build the pywin32).
-    python -m pip install pywin32
+    # python -m pip install pywin32
 
     # PyMI setup can be skipped once the upstream version is published on pypi
     Install-PyMI $PyMiRepoUrl

From 0779fa5647afee029439ba0d485b4deae15fc4e8 Mon Sep 17 00:00:00 2001
From: Adrian Vladu <avladu@cloudbasesolutions.com>
Date: Mon, 13 Apr 2020 18:02:00 +0300
Subject: [PATCH 19/25] Added how to for Python from source / embedded

Change-Id: Icd7de2a5d9f8bc001d9206574724df049de2a075
---
 scripts/Readme.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/scripts/Readme.md b/scripts/Readme.md
index aed6bdeb..efb47474 100644
--- a/scripts/Readme.md
+++ b/scripts/Readme.md
@@ -27,6 +27,7 @@
 
 
 ```powershell
+# full command line
 .\build_install_from_sources.ps1 `
     -CloudbaseInitRepoUrl "https://github.com/cloudbase/cloudbase-init" `
     -PyWin32RepoUrl "https://github.com/mhammond/pywin32" `
@@ -39,6 +40,11 @@
     -CleanBuildArtifacts:$false,
     -BuildDir "build"
 
+# install using Python from source (tag v3.7.7)
+.\build_install_from_sources.ps1 -PythonOrigin FromSource -PythonVersion "v3.7.7"
+
+# install using Python Embedded 3.7.7
+.\build_install_from_sources.ps1 -PythonOrigin Embedded -PythonVersion "3.7.7"
 ```
 
 

From bf6746e87a669c9ae0780c7d327c5a689679d153 Mon Sep 17 00:00:00 2001
From: Adrian Vladu <avladu@cloudbasesolutions.com>
Date: Mon, 13 Apr 2020 18:14:32 +0300
Subject: [PATCH 20/25] Added base usage

Change-Id: I4d017ff3e6b549445cb0cf27a672dce9969acb3f
---
 scripts/Readme.md | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/scripts/Readme.md b/scripts/Readme.md
index efb47474..cbf1ee99 100644
--- a/scripts/Readme.md
+++ b/scripts/Readme.md
@@ -40,10 +40,13 @@
     -CleanBuildArtifacts:$false,
     -BuildDir "build"
 
-# install using Python from source (tag v3.7.7)
+# install Cloudbase-Init using Python already installed (Python and Python scripts folders should be added to path).
+.\build_install_from_sources.ps1
+
+# install Cloudbase-Init using Python from source (tag v3.7.7)
 .\build_install_from_sources.ps1 -PythonOrigin FromSource -PythonVersion "v3.7.7"
 
-# install using Python Embedded 3.7.7
+# install Cloudbase-Init using Python Embedded 3.7.7
 .\build_install_from_sources.ps1 -PythonOrigin Embedded -PythonVersion "3.7.7"
 ```
 

From e15079c0a1cbb0da89dee0fe8778a479982a1595 Mon Sep 17 00:00:00 2001
From: Adrian Vladu <avladu@cloudbasesolutions.com>
Date: Mon, 13 Apr 2020 19:27:28 +0300
Subject: [PATCH 21/25] Use python directly for install and cleanup pip cache
 dir

Change-Id: I5c61d1919de5b6fa78ba6bd963337e4bfc17b753
---
 scripts/build_install_from_sources.ps1 | 29 ++++++++++++++++++--------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/scripts/build_install_from_sources.ps1 b/scripts/build_install_from_sources.ps1
index c5a88dd8..56079d78 100644
--- a/scripts/build_install_from_sources.ps1
+++ b/scripts/build_install_from_sources.ps1
@@ -160,16 +160,17 @@ function Install-PythonPackage {
 
     Write-Host "Installing Python package from ${SourcePath}"
 
-    $cmdArgs = @("-W ignore", "-m", "pip", "install")
-    if ($BuildWithoutBinaries) {
-        $cmdArgs += $PIP_BUILD_NO_BINARIES_ARGS
-    }
-    $cmdArgs += "."
-
     Run-CmdWithRetry {
         try {
             Push-Location (Join-Path $BuildDir $SourcePath)
-            Run-Command -Cmd "python" -Arguments $cmdArgs
+            if ($BuildWithoutBinaries) {
+                & python -m pip install --no-binary :all: .
+            } else {
+                & python -m pip install .
+            }
+            if ($LastExitCode) {
+                throw "Failed to install python package $SourcePath"
+            }
         } finally {
             Pop-Location
         }
@@ -392,7 +393,15 @@ function Clean-BuildArtifacts {
     Get-ChildItem -Include @("*.lib", "*.exp") -Exclude "python*" "$PythonDir\*" | Remove-Item -Force
 }
 
-
+function Clean-PipCacheDir {
+    if ($env:APPDATA) {
+        $pipCacheDir = Join-Path (Split-Path -Parent $env:APPDATA) "Local\pip"
+        if (Test-Path $pipCacheDir) {
+            Write-Host "Cleaning pip cache dir $pipCacheDir"
+            Remove-Item -Recurse -Force $pipCacheDir
+        }
+    }
+}
 ### Main ###
 
 try {
@@ -409,6 +418,7 @@ try {
         $PythonDir = Join-Path $BuildDir "python"
     }
     Prepare-BuildDir
+    Clean-PipCacheDir
 
     # Make sure VS 2015 is used
     Set-VCVars -Version "14.0"
@@ -426,9 +436,10 @@ try {
             Setup-FromSourcePythonEnvironment $PythonVersion
         }
         $env:path = "${PythonDir};${PythonDir}\scripts;" + $env:path
-        Install-SetuptoolsFromSource $SetuptoolsGitUrl
     }
 
+    Install-SetuptoolsFromSource $SetuptoolsGitUrl
+
     # Install PyWin32 from source
     Install-PyWin32FromSource $PyWin32RepoUrl
     # TODO. Comment the following line and uncomment the line before. Keep this line for faster script testing (it takes more than 10 minutes to build the pywin32).

From e77d75819b0aa4c01d6d24473f7f747d10be3ce9 Mon Sep 17 00:00:00 2001
From: Adrian Vladu <avladu@cloudbasesolutions.com>
Date: Tue, 14 Apr 2020 01:19:54 +0300
Subject: [PATCH 22/25] Removed Embedded python install

Change-Id: I17fdf4d88b4b847f21468027528fb2beefe8c047
---
 scripts/Readme.md                      |  26 +++---
 scripts/build_install_from_sources.ps1 | 111 ++-----------------------
 2 files changed, 19 insertions(+), 118 deletions(-)

diff --git a/scripts/Readme.md b/scripts/Readme.md
index cbf1ee99..0e3a98c3 100644
--- a/scripts/Readme.md
+++ b/scripts/Readme.md
@@ -1,6 +1,6 @@
 ### Requirements ###
 
-#### Clean Windows Server 2016 /2019 or Windows 10 install with latest updates
+#### Clean Windows Server 2016 / 2019 or Windows 10 install with latest updates
 
 #### Python installed and added to path (with pip installed) if PythonOrigin is set to AlreadyInstalled.
     Download link: https://www.python.org/ftp/python/3.7.7/python-3.7.7-amd64.exe
@@ -9,24 +9,28 @@
     - Programming Languages -> Visual C++ (all)
     - Windows and Web Development -> Windows 8.1 (only Tools and Windows SDKs)
 
-    VS 2015 Download link (you need to be logged in to access it): https://download.my.visualstudio.com/db/en_visual_studio_community_2015_with_update_1_x86_x64_web_installer_8234321.exe
+    VS 2015 Download page (you need to be logged in to access it): https://visualstudio.microsoft.com/vs/older-downloads/
     If you prefer to not build pywin32, Visual Studio 2017 / 2019 can be used too.
     Reboot after VS installation is complete.
 
 #### To build Python from source, Visual Studio 2017 Community installed with the following components:
     - VC++ 2017 v141
-    - Windows 10 SDK (10.0.14393.0). This version supports Python 3.8 build too.
+    - Windows 10 SDK (10.0.14393.0). This version of VS supports Python 3.8 build too.
 
 #### All the Python packages will be built and installed using pip flags: "--no-binary :all:"
 
-#### The full build of cloudbase-init (and dependencies) using Embedded Python takes around 10 minutes.
-
 #### The full build of cloudbase-init (and dependencies) using Python from source takes around 20 minutes.
 
 #### How to run:
 
 
 ```powershell
+# install Cloudbase-Init using Python already installed (Python and Python scripts folders should be added to path).
+.\build_install_from_sources.ps1
+
+# install Cloudbase-Init using Python from source (tag v3.7.7)
+.\build_install_from_sources.ps1 -PythonOrigin FromSource -PythonVersion "v3.7.7"
+
 # full command line
 .\build_install_from_sources.ps1 `
     -CloudbaseInitRepoUrl "https://github.com/cloudbase/cloudbase-init" `
@@ -39,15 +43,6 @@
     -WheelSourceUrl "https://github.com/pypa/wheel/archive/0.34.2.tar.gz" `,
     -CleanBuildArtifacts:$false,
     -BuildDir "build"
-
-# install Cloudbase-Init using Python already installed (Python and Python scripts folders should be added to path).
-.\build_install_from_sources.ps1
-
-# install Cloudbase-Init using Python from source (tag v3.7.7)
-.\build_install_from_sources.ps1 -PythonOrigin FromSource -PythonVersion "v3.7.7"
-
-# install Cloudbase-Init using Python Embedded 3.7.7
-.\build_install_from_sources.ps1 -PythonOrigin Embedded -PythonVersion "3.7.7"
 ```
 
 
@@ -56,8 +51,7 @@
    - Create / clean temporary build directory
    - If PythonOrigin="AlreadyInstalled" is set, do nothing. Python should be already installed and added to path.
    - If PythonOrigin="FromSource" is set, download the Python source from GitHub, build, prepare it and add it to path. Setuptools, pip and wheel will be built from source and installed.
-   - If PythonOrigin="Embedded" is set, download the Python embedded, prepare it and add it to path. Setuptools, pip and wheel will be built from source and installed.
    - Build and install PyWin32 from sources
    - Build and install PyMI from sources
    - Build, install and create Cloudbase-Init binary from sources
-   - If CleanBuildArtifacts is set and PythonOrigin is Embedded or FromSource, cleanup the .pdb, .pyc, header files.
+   - If CleanBuildArtifacts is set and PythonOrigin is FromSource, cleanup the .pdb, .pyc, header files.
diff --git a/scripts/build_install_from_sources.ps1 b/scripts/build_install_from_sources.ps1
index 56079d78..2933b2c5 100644
--- a/scripts/build_install_from_sources.ps1
+++ b/scripts/build_install_from_sources.ps1
@@ -3,19 +3,18 @@ param(
     [string]$CloudbaseInitRepoUrl="https://github.com/cloudbase/cloudbase-init",
     [string]$PyWin32RepoUrl="https://github.com/mhammond/pywin32",
     [string]$PyMiRepoUrl="https://github.com/cloudbase/PyMI",
-    [ValidateSet("AlreadyInstalled", "Embedded", "FromSource")]
+    [ValidateSet("AlreadyInstalled", "FromSource")]
     [string]$PythonOrigin="AlreadyInstalled",
-    # Specifies which version to use in case Embedded or FromSource is used
-    # In Embedded case: "https://www.python.org/ftp/python/$PythonVersion" should exist
+    # Specifies which version to use in case FromSource is used
     # In FromSource case, the GitHub https://github.com/python/cpython branch should exist (tags also work as branches)
     [string]$PythonVersion="v3.7.7",
-    # Specifies which setuptools git source to use in case Embedded or FromSource is used
+    # Specifies which setuptools git source to use in case FromSource is used
     [string]$SetuptoolsGitUrl="https://github.com/pypa/setuptools",
-    # Specifies which pip source in ttar.gz format to use in case Embedded or FromSource is used
+    # Specifies which pip source in tar.gz format to use in case FromSource is used
     [string]$PipSourceUrl="https://github.com/pypa/pip/archive/20.0.2.tar.gz",
-    # Specifies which wheel source in ttar.gz format to use in case Embedded or FromSource is used
+    # Specifies which wheel source in tar.gz format to use in case FromSource is used
     [string]$WheelSourceUrl="https://github.com/pypa/wheel/archive/0.34.2.tar.gz",
-    # If the PythonOrigin is set to Embedded or FromSource, clean the Python folder (remove .pdb, .pyc, header files)
+    # If the PythonOrigin is set to FromSource, clean the Python folder (remove .pdb, .pyc, header files)
     [switch]$CleanBuildArtifacts,
     [string]$BuildDir=""
 )
@@ -193,23 +192,6 @@ function Expand-Archive {
     }
 }
 
-function Setup-PythonPackage {
-    param([string]$SourcePath)
-
-    Write-Host "Setup Python package from ${SourcePath}"
-
-    $cmdArgs = @("-W ignore", "setup.py", "install")
-
-    Run-CmdWithRetry {
-        try {
-            Push-Location (Join-Path $BuildDir $SourcePath)
-            Run-Command -Cmd "python" -Arguments $cmdArgs
-        } finally {
-            Pop-Location
-        }
-    }
-}
-
 function Prepare-BuildDir {
     Write-Host "Creating / Cleaning up build directory ${BuildDir}"
 
@@ -239,15 +221,6 @@ function Install-PyWin32FromSource {
     Install-PythonPackage -SourcePath $sourcePath
 }
 
-function Install-ComtypesFromSource {
-    param($Url)
-
-    $sourcePath = "comptype"
-
-    Clone-Repo $Url $sourcePath
-    Install-PythonPackage -SourcePath $sourcePath
-}
-
 function Install-PyMI {
     param($Url)
 
@@ -291,68 +264,6 @@ function Install-SetuptoolsFromSource {
     }
 }
 
-function Setup-EmbeddedPythonEnvironment {
-    param($EmbeddedPythonVersion)
-
-    $EmbeddedPythonUrl = "https://www.python.org/ftp/python/${EmbeddedPythonVersion}/python-${EmbeddedPythonVersion}-embed-amd64.zip"
-    $SourcePythonUrl = "https://www.python.org/ftp/python/${EmbeddedPythonVersion}/Python-${EmbeddedPythonVersion}.tgz"
-    $pythonVersionHeader = "python" + (($EmbeddedPythonVersion.split(".") | Select-Object -First 2) -Join  "")
-
-    Download-File $EmbeddedPythonUrl "${PythonDir}.zip"
-    New-Item -Type Directory -Path $PythonDir
-    Expand-Archive "${PythonDir}.zip" $PythonDir
-    Remove-Item -Force "${PythonDir}.zip"
-
-    $sourcePythonDir = "$BuildDir\source-python"
-    Download-File "${SourcePythonUrl}" "${sourcePythonDir}.tgz"
-    New-Item -Type Directory -Path $sourcePythonDir
-    Expand-Archive "${sourcePythonDir}.tgz" $sourcePythonDir
-    Remove-Item -Force "${sourcePythonDir}.tgz"
-    New-Item -Type Directory -Path "$sourcePythonDir\src"
-    Expand-Archive "${sourcePythonDir}\source-python.tar" "$sourcePythonDir\src"
-    Remove-Item -Force -Recurse "${sourcePythonDir}\source-python.tar"
-    $sourcePythonDir = "${sourcePythonDir}\src\Python-${EmbeddedPythonVersion}"
-
-    Remove-Item -Force "${PythonDir}\${pythonVersionHeader}._pth"
-
-    New-Item -Type Directory -Path "${PythonDir}\Lib"
-    Expand-Archive "${PythonDir}\${pythonVersionHeader}.zip" "${PythonDir}\Lib"
-    Remove-Item -Force "${PythonDir}\${pythonVersionHeader}.zip"
-
-    Copy-Item -Recurse -Force "${sourcePythonDir}\Include" "${PythonDir}\"
-    Copy-Item -Recurse -Force "${sourcePythonDir}\PC\pyconfig.h" "${PythonDir}\Include\"
-
-    New-Item -Type Directory -Path "${PythonDir}\libs\"
-    $pythonLibPath = "${PythonDir}\libs\${pythonVersionHeader}.lib"
-    $pythonLibDefRawPath = "${PythonDir}\libs\${pythonVersionHeader}.raw.def"
-    $pythonLibDefPath = "${PythonDir}\libs\${pythonVersionHeader}.def"
-    $pythonDllPath = "${PythonDir}\${pythonVersionHeader}.dll"
-    dumpbin.exe /exports "${pythonDllPath}" > "${pythonLibDefRawPath}"
-    if ($LastExitCode) {
-        throw "Failed to dump symbols"
-    }
-    Write-Output "EXPORTS" > "${pythonLibDefPath}"
-    Get-Content "${pythonLibDefRawPath}" | ForEach-Object {
-        $splitObject = $_.split(" ")
-        [array]::reverse($splitObject)
-        if ($splitObject[0] -clike "*Py*") {
-            Write-Output $splitObject[0] >> "${pythonLibDefPath}"
-        }
-    }
-    lib.exe /def:"${pythonLibDefPath}" /out:"${pythonLibPath}" /machine:x64
-    if ($LastExitCode) {
-        throw "Failed to create symbol map"
-    }
-    Remove-Item -Force $pythonLibDefRawPath
-    Remove-Item -Force $pythonLibDefPath
-
-    # Embedded Python has bdist_wininst.pyc reimplemented to throw an error if any package using this feature
-    # is to be installed. Pywin32 and comtypes packages cannot be installed with pip or by running setup.py install.
-    $bdistFile = "Lib\distutils\command\bdist_wininst.py"
-    Copy-Item -Force "${sourcePythonDir}\${bdistFile}" "${PythonDir}\${bdistFile}"
-    Remove-Item -Force "${PythonDir}\${bdistFile}c"
-}
-
 function Setup-FromSourcePythonEnvironment {
     param($FromSourcePythonVersion)
 
@@ -426,15 +337,11 @@ try {
     # Setup pip upper requirements
     Setup-PythonPip
 
-    if (@("Embedded", "FromSource") -contains $PythonOrigin) {
+    if (@("FromSource") -contains $PythonOrigin) {
         if (!($PythonVersion -and $SetuptoolsGitUrl -and $PipSourceUrl -and $WheelSourceUrl)) {
             throw "If PythonOrigin is set to ${PythonOrigin}, SetuptoolsGitUrl, PipSourceUrl and WheelSourceUrl must be set"
         }
-        if ($PythonOrigin -eq "Embedded") {
-            Setup-EmbeddedPythonEnvironment $PythonVersion
-        } else {
-            Setup-FromSourcePythonEnvironment $PythonVersion
-        }
+        Setup-FromSourcePythonEnvironment $PythonVersion
         $env:path = "${PythonDir};${PythonDir}\scripts;" + $env:path
     }
 
@@ -450,7 +357,7 @@ try {
 
     Install-CloudbaseInit $CloudbaseInitRepoUrl
 
-    if ($CleanBuildArtifacts -and (@("Embedded", "FromSource") -contains $PythonOrigin)) {
+    if ($CleanBuildArtifacts -and (@("FromSource") -contains $PythonOrigin)) {
         Clean-BuildArtifacts
     }
 } finally {

From 315d643cc3054ed8c46cf51a611fafcbf62df5a3 Mon Sep 17 00:00:00 2001
From: Adrian Vladu <avladu@cloudbasesolutions.com>
Date: Tue, 14 Apr 2020 14:02:52 +0300
Subject: [PATCH 23/25] Set Windows SDK to 8.1 using vcvarsall

Change-Id: I4703b2c41ac071de82096c22fac8d6211cfcb38b
---
 scripts/build_install_from_sources.ps1 | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/scripts/build_install_from_sources.ps1 b/scripts/build_install_from_sources.ps1
index 2933b2c5..bd83b497 100644
--- a/scripts/build_install_from_sources.ps1
+++ b/scripts/build_install_from_sources.ps1
@@ -72,14 +72,15 @@ function Download-File {
 function Set-VCVars {
     param(
         $Version="14.0",
-        $Platform="x86_amd64"
+        $Platform="x86_amd64",
+        $Sdk="8.1"
     )
 
     Write-Host "Setting Visual Studio version ${Version} environment variables"
 
     Push-Location "$ENV:ProgramFiles (x86)\Microsoft Visual Studio ${Version}\VC\"
     try {
-        cmd /c "vcvarsall.bat $platform & set" |
+        cmd /c "vcvarsall.bat $platform $Sdk & set" |
             ForEach-Object {
                 if ($_ -match "=") {
                     $v = $_.split("=")
@@ -331,8 +332,8 @@ try {
     Prepare-BuildDir
     Clean-PipCacheDir
 
-    # Make sure VS 2015 is used
-    Set-VCVars -Version "14.0"
+    # Make sure VS 2015 and Windows 8.1 SDK are used
+    Set-VCVars -Version "14.0" -Sdk "8.1"
 
     # Setup pip upper requirements
     Setup-PythonPip

From 1a1ba6f31146afdea576f92534ac511812488a32 Mon Sep 17 00:00:00 2001
From: Adrian Vladu <avladu@cloudbasesolutions.com>
Date: Tue, 14 Apr 2020 15:15:26 +0300
Subject: [PATCH 24/25] Add parameter to remove unnecessary executables

Change-Id: I962c304f89dd346a7216c546bfc39d3575d31374
---
 scripts/Readme.md                      |  2 ++
 scripts/build_install_from_sources.ps1 | 27 ++++++++++++++++++++++++--
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/scripts/Readme.md b/scripts/Readme.md
index 0e3a98c3..36198554 100644
--- a/scripts/Readme.md
+++ b/scripts/Readme.md
@@ -42,6 +42,7 @@
     -PipSourceUrl "https://github.com/pypa/pip/archive/20.0.2.tar.gz" `
     -WheelSourceUrl "https://github.com/pypa/wheel/archive/0.34.2.tar.gz" `,
     -CleanBuildArtifacts:$false,
+    -RemoveUnnecessaryExecutables:$false,
     -BuildDir "build"
 ```
 
@@ -55,3 +56,4 @@
    - Build and install PyMI from sources
    - Build, install and create Cloudbase-Init binary from sources
    - If CleanBuildArtifacts is set and PythonOrigin is FromSource, cleanup the .pdb, .pyc, header files.
+   - If RemoveUnnecessaryExecutables is set and PythonOrigin is FromSource, remove the .exe files (excluding python.exe, setuptools/pip/wheel and cloudbase-init.exe).
diff --git a/scripts/build_install_from_sources.ps1 b/scripts/build_install_from_sources.ps1
index bd83b497..f450f223 100644
--- a/scripts/build_install_from_sources.ps1
+++ b/scripts/build_install_from_sources.ps1
@@ -16,6 +16,7 @@ param(
     [string]$WheelSourceUrl="https://github.com/pypa/wheel/archive/0.34.2.tar.gz",
     # If the PythonOrigin is set to FromSource, clean the Python folder (remove .pdb, .pyc, header files)
     [switch]$CleanBuildArtifacts,
+    [switch]$RemoveUnnecessaryExecutables,
     [string]$BuildDir=""
 )
 
@@ -295,6 +296,8 @@ function Setup-FromSourcePythonEnvironment {
 }
 
 function Clean-BuildArtifacts {
+    Write-Host "Cleaning build artifacts"
+
     # Remove the Include folder
     Remove-Item -Force -Recurse "${PythonDir}\Include"
     # Remove all the __pycache__ folders
@@ -305,6 +308,21 @@ function Clean-BuildArtifacts {
     Get-ChildItem -Include @("*.lib", "*.exp") -Exclude "python*" "$PythonDir\*" | Remove-Item -Force
 }
 
+function Remove-UnnecessaryExecutables {
+    $excludeList = @("python.exe", "cloudbase-init*",
+                     "pip*", "easy_install*", "wheel*",
+                     "t64.exe", "w64.exe", "t32.exe", "w32.exe")
+
+    Write-Host "Removing unnecessary scripts and executables"
+
+    # Remove unnecessary scripts
+    Get-ChildItem -Exclude $excludeList "$PythonDir\Scripts\" | `
+        Remove-Item -Force -ErrorAction SilentlyContinue
+    # Remove unnecessary executables
+    Get-ChildItem -Recurse -Include "*.exe" -Exclude $excludeList $PythonDir | `
+        Remove-Item -Force -ErrorAction SilentlyContinue
+}
+
 function Clean-PipCacheDir {
     if ($env:APPDATA) {
         $pipCacheDir = Join-Path (Split-Path -Parent $env:APPDATA) "Local\pip"
@@ -358,8 +376,13 @@ try {
 
     Install-CloudbaseInit $CloudbaseInitRepoUrl
 
-    if ($CleanBuildArtifacts -and (@("FromSource") -contains $PythonOrigin)) {
-        Clean-BuildArtifacts
+    if (@("FromSource") -contains $PythonOrigin) {
+        if ($CleanBuildArtifacts) {
+            Clean-BuildArtifacts
+        }
+        if ($RemoveUnnecessaryExecutables) {
+            Remove-UnnecessaryExecutables
+        }
     }
 } finally {
     $endDate = Get-Date

From 2935d70904da4fc3fd77d8df6e2996f6f5e8fe60 Mon Sep 17 00:00:00 2001
From: Adrian Vladu <avladu@cloudbasesolutions.com>
Date: Thu, 30 Apr 2020 11:07:19 +0300
Subject: [PATCH 25/25] Add flag to remove adodbapi from pywin32 code

Change-Id: Ia4c8719490b7b3986b9113955dcf41136e432663
---
 scripts/Readme.md                      |  2 ++
 scripts/build_install_from_sources.ps1 | 15 ++++++++++++++-
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/scripts/Readme.md b/scripts/Readme.md
index 36198554..63f64963 100644
--- a/scripts/Readme.md
+++ b/scripts/Readme.md
@@ -43,6 +43,7 @@
     -WheelSourceUrl "https://github.com/pypa/wheel/archive/0.34.2.tar.gz" `,
     -CleanBuildArtifacts:$false,
     -RemoveUnnecessaryExecutables:$false,
+    -RemovePyWin32Adodbapi:$false,
     -BuildDir "build"
 ```
 
@@ -57,3 +58,4 @@
    - Build, install and create Cloudbase-Init binary from sources
    - If CleanBuildArtifacts is set and PythonOrigin is FromSource, cleanup the .pdb, .pyc, header files.
    - If RemoveUnnecessaryExecutables is set and PythonOrigin is FromSource, remove the .exe files (excluding python.exe, setuptools/pip/wheel and cloudbase-init.exe).
+   - If RemovePyWin32Adodbapi is set, remove the Adodbapi sources and references from PyWin32 source code
diff --git a/scripts/build_install_from_sources.ps1 b/scripts/build_install_from_sources.ps1
index f450f223..0f6e2a54 100644
--- a/scripts/build_install_from_sources.ps1
+++ b/scripts/build_install_from_sources.ps1
@@ -17,6 +17,7 @@ param(
     # If the PythonOrigin is set to FromSource, clean the Python folder (remove .pdb, .pyc, header files)
     [switch]$CleanBuildArtifacts,
     [switch]$RemoveUnnecessaryExecutables,
+    [switch]$RemovePyWin32Adodbapi,
     [string]$BuildDir=""
 )
 
@@ -219,7 +220,19 @@ function Install-PyWin32FromSource {
 
     $sourcePath = "pywin32"
 
-    Clone-Repo $Url $sourceFolder
+    Clone-Repo $Url $sourcePath
+    if ($RemovePyWin32Adodbapi) {
+        try{
+            Write-Host "Remove PyWin32Adodbapi sources and references"
+            Push-Location "${BuildDir}/${sourcePath}"
+            Remove-Item -Force -Recurse "adodbapi"
+            Get-Content "setup.py" | Where-Object {!($_ -like "*adodbapi*")} | `
+                Out-File "setup.py.bak" -Encoding ascii
+            Move-Item -Force "setup.py.bak" "setup.py"
+        } finally {
+            Pop-Location
+        }
+    }
     Install-PythonPackage -SourcePath $sourcePath
 }