Fix Nagler tests to run on Linux (#524)

* Author: William Xie <willxie@microsoft.com>
Date:   Mon Oct 10 11:23:07 2022 -0700

    Squashed commit of the following:

    commit 2292449
    Author: William Xie <willxie@microsoft.com>
    Date:   Fri Oct 7 14:30:06 2022 -0700

        Fix up some last bugs

    commit 364ab15
    Author: William Xie <willxie@microsoft.com>
    Date:   Fri Oct 7 10:36:24 2022 -0700

        Fix libxml2 handling of comment nodes

    commit 4bcbe1f
    Author: William Xie <willxie@microsoft.com>
    Date:   Thu Oct 6 21:42:24 2022 -0700

        Add timeout override

    commit 85b5661
    Author: William Xie <willxie@microsoft.com>
    Date:   Thu Oct 6 21:22:28 2022 -0700

        Fix logging test

    commit 8ca135c
    Author: William Xie <willxie@microsoft.com>
    Date:   Thu Oct 6 21:02:01 2022 -0700

        Fix linux nagler tests

    commit e21c6db
    Author: William Xie <willxie@microsoft.com>
    Date:   Thu Oct 6 11:33:38 2022 -0700

        temporary changes

* Update test.md and fix linux managed build

* Test PR build

* PR comments

Author: willxie-eng

build/yaml/jobs/linux/tests.yaml

@@ -0,0 +1,65 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+# Job Template: Test Linux Binaries
+
+parameters:
+  MatrixStrategy: {}
+
+jobs:
+- job: Test_Linux_Binaries
+  displayName: Linux Tests
+
+  strategy:
+    matrix:
+      ${{ insert }}: ${{ parameters.MatrixStrategy }}
+
+  pool:
+    ${{ if eq(variables['System.TeamProject'], 'devdiv') }}:
+      name: AzurePipelines-EO
+      demands: ImageOverride -equals AzurePipelinesUbuntu20.04compliant
+    ${{ else }}:
+      vmImage: ubuntu-latest
+
+  workspace:
+    clean: outputs
+
+  variables:
+    configuration: Debug
+
+  steps:
+  - template: ../../steps/azuredevops/downloadbuildartifacts.yaml
+    parameters:
+      ArtifactName: binaries-windows-$(Configuration)
+
+  - template: ../../steps/azuredevops/downloadbuildartifacts.yaml
+    parameters:
+      ArtifactName: binaries-linux-ubuntu-$(Configuration)
+
+  # Simulate DeploymentItem by copying the Linux binaries
+  - task: CopyFiles@2
+    displayName: Setup Linux binaries for tests
+    inputs:
+      sourceFolder: $(Build.StagingDirectory)/binaries-linux-ubuntu-$(Configuration)/ClrInstrumentationEngine/
+      targetFolder: $(Build.StagingDirectory)/binaries-windows-$(Configuration)/AnyCPU/$(TargetFrameworkFolder)/
+
+  - task: UseDotNet@2
+    displayName: 'Use .NET sdk'
+    inputs:
+      version: $(NetSdkVersion)
+      includePreviewVersions: true
+      installationPath: $(Agent.TempDirectory)/dotnet
+
+  # Environment variable used by tests
+  - bash: |
+      echo "##vso[task.setvariable variable=DOTNET_EXE]$(Agent.TempDirectory)/dotnet/dotnet"
+
+  - task: DotNetCoreCLI@2
+    displayName: 'Run Tests: InstrEngineTests.dll'
+    inputs:
+      command: test
+      workingDirectory: $(Agent.TempDirectory)/dotnet
+      arguments: $(Build.StagingDirectory)/binaries-windows-$(Configuration)/AnyCPU/$(TargetFrameworkFolder)/InstrEngineTests.dll
+      testRunTitle: Linux Nagler Tests
+      publishTestResults: true
+    condition: succeededOrFailed()

build/yaml/jobs/tests.yaml

@@ -36,5 +36,13 @@ jobs:
           Platform: 'AnyCPU'
           RunSettingsFile: 'x64.runsettings'
 
-# Linux Binaries (TBD)
-#- template: linux/tests.yaml
+# Test Linux Binaries
+- template: linux/tests.yaml
+  parameters:
+    MatrixStrategy:
+      NET_Next:
+        TargetFrameworkFolder: net70
+        NetSdkVersion: 7.x
+      NET_LTS:
+        TargetFrameworkFolder: net60
+        NetSdkVersion: 6.x

build/yaml/steps/azuredevops/downloadbuildartifacts.yaml

@@ -6,6 +6,7 @@
 parameters:
   ArtifactName: ''
   ItemPattern: '**'
+  DownloadPath: $(Build.ArtifactStagingDirectory)
 
 steps:
 # download Artifacts
@@ -14,5 +15,5 @@ steps:
   inputs:
     downloadType: single
     artifactName: ${{ parameters.ArtifactName }}
-    downloadPath: $(Build.ArtifactStagingDirectory)
+    downloadPath: ${{ parameters.DownloadPath }}
     itemPattern: ${{ parameters.ItemPattern }}

docs/test.md

@@ -35,5 +35,22 @@ Verbose|Sets msbuild verbosity to `normal`. By default, verbosity is set to `Err
 
 ### Linux
 
-Currently there are no unit tests support for Linux. However, Linux builds will run via PR validations. Please contact
-clrieowners@microsoft.com for more details.
+#### Prerequisites
+
+The following technologies/tools are required in order to run Linux tests on Windows. These instructions were tested for Ubuntu 18.04 LTS and may require adjustments for other distros/versions.
+
+* WSL (Windows Subsystem for Linux) 2.0 with Ubuntu 18.04 LTS
+  - WSL is available on Windows 10 Build 19041+ and Windows 11.
+  - Run this in a command prompt `wsl --install -d Ubuntu-18.04` or download Ubuntu 18.04 from the Microsoft Store. Restart as needed and setup a user/pass for your account.
+* Docker Desktop for Linux (Ubuntu)
+  - Follow the instructions from the official docs https://docs.docker.com/engine/install/ubuntu/
+
+#### Steps
+
+1. On Windows, build InstrumentationEngine for Debug AnyCPU (which builds managed projects and dlls) 
+  - You can also run `dotnet build InstrumentationEngine.sln` in the interactive container session from step 2 instead. 
+2. Run `src\scripts\DockerLocalBuild.ps1 -Wsl -BuildDockerImage -Interactive -RebuildImage`. This will build a docker container and drop you in an interactive session. By default builds for Ubuntu/Debian; set `-CLib musl` if you want to build for Alpine Linux instead.
+3. Run `./src/build.sh` within the interactive session. This builds the native shared object files (.so).
+4. Run `dotnet test bin/Debug/AnyCPU/net70/InstrEngineTests.dll`
+
+Please contact clrieowners@microsoft.com for any details or questions.

src/Common.Lib/XmlNode.cpp

@@ -82,7 +82,15 @@ namespace CommonLib
         name = pName.m_str;
 
     #else
-        IfFailRet(SystemString::Convert((const char*)m_pNode->name, name));
+        // LibXML2 will return an empty string for comment node names so we mark it as '#comment'.
+        if (m_pNode->type == XML_COMMENT_NODE)
+        {
+            name = _T("#comment");
+        }
+        else
+        {
+            IfFailRet(SystemString::Convert((const char*)m_pNode->name, name));
+        }
     #endif
 
         return S_OK;

src/InstrumentationEngine.Attach/InstrumentationEngine.Attach.csproj

@@ -42,7 +42,11 @@
     <PackageReference Include="System.CommandLine" Version="2.0.0-beta1.20104.2" />
   </ItemGroup>
   <Import Project="$(EnlistmentRoot)\build\Managed.targets" />
-  <Target Name="XsdGeneration" AfterTargets="PrepareForBuild" Inputs="@(XsdSchemaFile)" Outputs="@(XsdSchemaFile.OutputFile)" Condition="'@(XsdSchemaFile)' != ''">
+  <Target Name="XsdGeneration" 
+          AfterTargets="PrepareForBuild" 
+          Inputs="@(XsdSchemaFile)" 
+          Outputs="@(XsdSchemaFile.OutputFile)" 
+          Condition="$([MSBuild]::IsOSPlatform('Windows')) AND '@(XsdSchemaFile)' != ''">
     <Error Condition="'%(XsdSchemaFile.Namespace)' == ''" Text="Missing Namespace attribute for XsdSchemaFile %(XsdSchemaFile.FullPath)" />
     <PropertyGroup>
       <XsdToolPath Condition="!Exists('$(XsdToolPath)')">$(MSBuildProgramFiles32)\Microsoft SDKs\Windows\v10.0A\bin\NETFX 4.8 Tools\xsd.exe</XsdToolPath>

src/InstrumentationEngine/ProfilerManagerForInstrumentationMethod.cpp

@@ -24,9 +24,15 @@ CProfilerManagerForInstrumentationMethod::CProfilerManagerForInstrumentationMeth
         // 1 offset = trim start bracket
         // 36 length = keep only Guid portion
         m_wszInstrumentationMethodGuid = tstring(wszCurrentMethodGuid).substr(1, 36);
+        for (int i = 0; i < 36; i++)
+        {
+            m_wszInstrumentationMethodGuid[i] = std::toupper(m_wszInstrumentationMethodGuid[i]);
+        }
+
         m_instrumentationMethodGuid = instrumentationMethodGuid;
 
         // Get InstrumentationMethod-specific LogLevel Environment Variable
+        // Environment variables are case-insensitive on Windows and case-sensitive on Linux/Mac.
         tstring wszInstrumentationMethodLogLevelEnvVar = _T("MicrosoftInstrumentationEngine_LogLevel_") + m_wszInstrumentationMethodGuid;
         WCHAR wszEnvVar[MAX_PATH];
         if (GetEnvironmentVariable(wszInstrumentationMethodLogLevelEnvVar.c_str(), wszEnvVar, MAX_PATH) > 0)

src/Tests/InstrEngineTests/InstrEngineTests/InstrEngineTests.csproj

@@ -20,6 +20,7 @@
     <EmbeddedResource Include="EmbeddedResources\**" />
   </ItemGroup>
   <ItemGroup>
+    <None Include="RefStructsTests.cs" />
     <None Remove="EmbeddedResources\InvalidCSharp\readme.txt" />
   </ItemGroup>
   <ItemGroup Condition="!$([MSBuild]::IsTargetFrameworkCompatible('$(TargetFramework)', 'net7.0'))">

src/Tests/InstrEngineTests/InstrEngineTests/MaxStackSizeTests.cs

@@ -21,6 +21,9 @@ namespace InstrEngineTests
     [DeploymentItem(PathUtils.NaglerInstrumentationConfigX86BinPath)]
     [DeploymentItem(PathUtils.NaglerInstrumentationMethodX64BinPath)]
     [DeploymentItem(PathUtils.NaglerInstrumentationMethodX86BinPath)]
+    [DeploymentItem(PathUtils.LinuxInstrumentationEngineX64BinPath)]
+    [DeploymentItem(PathUtils.LinuxNaglerInstrumentationConfigX64BinPath)]
+    [DeploymentItem(PathUtils.LinuxNaglerInstrumentationMethodX64BinPath)]
     public class MaxStackSizeTests
     {
         private static TestContext Context;

src/Tests/InstrEngineTests/InstrEngineTests/PathUtils.cs

@@ -13,15 +13,19 @@ public static class PathUtils
         private const string TestScriptsFolder = "TestScripts";
         private const string ResourcesFolder = "ExtractedResources";
 
-        // All paths are relative to the AnyCPU binaries directory e.g. bin\Debug\AnyCPU
-        public const string BaselinesBinPath = @".\" + BaselinesFolder;
-        public const string TestScriptsBinPath = @".\" + TestScriptsFolder;
-        public const string InstrumentationEngineX64BinPath = @"..\..\x64\MicrosoftInstrumentationEngine_x64.dll";
-        public const string InstrumentationEngineX86BinPath = @"..\..\x86\MicrosoftInstrumentationEngine_x86.dll";
-        public const string NaglerInstrumentationConfigX64BinPath = @"..\..\x64\NaglerInstrumentationMethod_x64.xml";
-        public const string NaglerInstrumentationConfigX86BinPath = @"..\..\x86\NaglerInstrumentationMethod_x86.xml";
-        public const string NaglerInstrumentationMethodX64BinPath = @"..\..\x64\NaglerInstrumentationMethod_x64.dll";
-        public const string NaglerInstrumentationMethodX86BinPath = @"..\..\x86\NaglerInstrumentationMethod_x86.dll";
+        // All paths are relative to the AnyCPU binaries directory e.g. bin/Debug/AnyCPU/(framework)
+        // Use forward slashes '/' to maintain compatibility between Windows and Linux
+        public const string BaselinesBinPath = @"./" + BaselinesFolder;
+        public const string TestScriptsBinPath = @"./" + TestScriptsFolder;
+        public const string InstrumentationEngineX64BinPath = @"../../x64/MicrosoftInstrumentationEngine_x64.dll";
+        public const string InstrumentationEngineX86BinPath = @"../../x86/MicrosoftInstrumentationEngine_x86.dll";
+        public const string NaglerInstrumentationConfigX64BinPath = @"../../x64/NaglerInstrumentationMethod_x64.xml";
+        public const string NaglerInstrumentationConfigX86BinPath = @"../../x86/NaglerInstrumentationMethod_x86.xml";
+        public const string NaglerInstrumentationMethodX64BinPath = @"../../x64/NaglerInstrumentationMethod_x64.dll";
+        public const string NaglerInstrumentationMethodX86BinPath = @"../../x86/NaglerInstrumentationMethod_x86.dll";
+        public const string LinuxInstrumentationEngineX64BinPath = @"../../../../out/Linux/bin/x64.Debug/ClrInstrumentationEngine/libInstrumentationEngine.so";
+        public const string LinuxNaglerInstrumentationConfigX64BinPath = @"../../../../out/Linux/bin/x64.Debug/ClrInstrumentationEngine/NaglerInstrumentationMethod/LinuxNaglerInstrumentationMethod.xml";
+        public const string LinuxNaglerInstrumentationMethodX64BinPath = @"../../../../out/Linux/bin/x64.Debug/ClrInstrumentationEngine/libNaglerInstrumentationEngine.so";
 
         public static string GetAssetsPath()
         {