Service Fabric cluster with 2 exposed https endpoint and different ports

Question

I created 2 Stateless Service Fabric services, that I need both exposed and be accessible from the web via https:

• Engine, (Asp.net Core API) exposed via HTTP on port 1212 and HTTPS on port 8465
• Website (Asp.net Core Web App) exposed via HTTPS on port 443

I'm for now LOCAL ONLY, using WebListener.

ServiceManifest.XML ENGINE

<?xml version="1.0" encoding="utf-8"?>
<ServiceManifest Name="EnginePkg"
                 Version="1.0.0"
                 xmlns="http://schemas.microsoft.com/2011/01/fabric"
                 xmlns:xsd="http://www.w3.org/2001/XMLSchema"
                 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
  <ServiceTypes>
    <StatelessServiceType ServiceTypeName="EngineType" />
  </ServiceTypes>

 <CodePackage Name="Code" Version="1.0.0">
    <EntryPoint>
      <ExeHost>
        <Program>Engine.exe</Program>
        <WorkingFolder>CodePackage</WorkingFolder>
      </ExeHost>
    </EntryPoint>
  </CodePackage>

 <ConfigPackage Name="Config" Version="1.0.0" />

  <Resources>
    <Endpoints>
      <Endpoint Protocol="http" Name="EngineEndpoint" Type="Input" Port="1212" />
      <Endpoint Protocol="https" Name="EngineEndpointSecure" Type="Input" Port="8465" />
    </Endpoints>
  </Resources>
</ServiceManifest>

ServiceManifest.XML WEBSITE

<?xml version="1.0" encoding="utf-8"?>
<ServiceManifest Name="WebsitePkg"
                 Version="1.0.0"
                 xmlns="http://schemas.microsoft.com/2011/01/fabric"
                 xmlns:xsd="http://www.w3.org/2001/XMLSchema"
                 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
  <ServiceTypes>
    <StatelessServiceType ServiceTypeName="WebsiteType" />
  </ServiceTypes>

  <CodePackage Name="Code" Version="1.0.0">
    <EntryPoint>
      <ExeHost>
        <Program>Website.exe</Program>
        <WorkingFolder>CodePackage</WorkingFolder>
      </ExeHost>
    </EntryPoint>
  </CodePackage>

  <ConfigPackage Name="Config" Version="1.0.0" />

  <Resources>
    <Endpoints>
    <Endpoint Protocol="https" Name="WebsiteEndpoint" Type="Input" Port="443" />
    </Endpoints>
  </Resources>
</ServiceManifest>

ENGINE.CS

internal sealed class Engine : StatelessService
    {
        public Engine(StatelessServiceContext context)
            : base(context)
        { }

        /// <summary>
        /// Optional override to create listeners (like tcp, http) for this service instance.
        /// </summary>
        /// <returns>The collection of listeners.</returns>
        protected override IEnumerable<ServiceInstanceListener> CreateServiceInstanceListeners()
        {
            return new ServiceInstanceListener[]
            {
                new ServiceInstanceListener(serviceContext =>
                    new WebListenerCommunicationListener(serviceContext, "EngineEndpoint", (url, listener) =>
                    {
                        ServiceEventSource.Current.ServiceMessage(serviceContext, $"Starting WebListener on {url}");

                        return new WebHostBuilder().UseWebListener()
                                    .ConfigureServices(
                                        services => services
                                            .AddSingleton(serviceContext))
                                    .UseContentRoot(Directory.GetCurrentDirectory())
                                    .UseStartup<Startup>()
                                    .UseApplicationInsights()
                                    .UseServiceFabricIntegration(listener, ServiceFabricIntegrationOptions.None)
                                    .UseUrls(url)
                                    .Build();
                    }), "EngineEndpoint"),//Name is important for multiple endpoints

               new ServiceInstanceListener(serviceContext =>
                    new WebListenerCommunicationListener(serviceContext, "EngineEndpointSecure", (url, listener) =>
                    {
                        ServiceEventSource.Current.ServiceMessage(serviceContext, $"Starting Secure WebListener on {url}");

                        return new WebHostBuilder().UseWebListener()
                                    .ConfigureServices(
                                        services => services
                                            .AddSingleton(serviceContext))
                                    .UseContentRoot(Directory.GetCurrentDirectory())
                                    .UseStartup<Startup>()
                                    .UseApplicationInsights()
                                    .UseServiceFabricIntegration(listener, ServiceFabricIntegrationOptions.None)
                                    .UseUrls(url)
                                    .Build();
                    }), "EngineEndpointSecure")
            };
        }
    }

WEBSITE.CS

internal sealed class Website : StatelessService
{
    public Website(StatelessServiceContext context)
        : base(context)
    { }

    /// <summary>
    /// Optional override to create listeners (like tcp, http) for this service instance.
    /// </summary>
    /// <returns>The collection of listeners.</returns>
    protected override IEnumerable<ServiceInstanceListener> CreateServiceInstanceListeners()
    {
        return new ServiceInstanceListener[]
        {
            new ServiceInstanceListener(serviceContext =>
                new WebListenerCommunicationListener(serviceContext, "WebsiteEndpoint", (url, listener) =>
                {
                    ServiceEventSource.Current.ServiceMessage(serviceContext, $"Starting WebListener on {url}");

                    return new WebHostBuilder().UseWebListener()
                                .ConfigureServices(
                                    services => services
                                        .AddSingleton(serviceContext))
                                .UseContentRoot(Directory.GetCurrentDirectory())
                                .UseStartup<Startup>()
                                .UseApplicationInsights()
                                .UseServiceFabricIntegration(listener, ServiceFabricIntegrationOptions.None)      
                                .UseUrls(url)
                                .Build();
                }), "WebsiteEndpoint"),
        };
    }
}

APPLICATIONMANIFEST.XML

<?xml version="1.0" encoding="utf-8"?>
<ApplicationManifest xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" ApplicationTypeName="ProjectSFType" ApplicationTypeVersion="1.0.0" xmlns="http://schemas.microsoft.com/2011/01/fabric">
  <Parameters>

    <!--STATELESS-->
    <Parameter Name="Engine_InstanceCount" DefaultValue="-1" />
    <Parameter Name="Website_InstanceCount" DefaultValue="-1" />

  </Parameters>
  <ServiceManifestImport>
    <ConfigOverrides />
    <Policies>
      <EndpointBindingPolicy EndpointRef="WebsiteEndpoint" CertificateRef="FabricFront" />
      <EndpointBindingPolicy EndpointRef="EngineEndpointSecure" CertificateRef="FabricFront" />
    </Policies>
  </ServiceManifestImport>
  <ServiceManifestImport>
    <ServiceManifestRef ServiceManifestName="EnginePkg" ServiceManifestVersion="1.0.0" />
    <ConfigOverrides />
  </ServiceManifestImport>
  <ServiceManifestImport>
    <ServiceManifestRef ServiceManifestName="WebsitePkg" ServiceManifestVersion="1.0.0" />
    <ConfigOverrides />
  </ServiceManifestImport>
  <DefaultServices>
    <Service Name="Engine">
      <StatelessService ServiceTypeName="EngineType" InstanceCount="[Engine_InstanceCount]">
        <SingletonPartition />
      </StatelessService>
    </Service>
    <Service Name="Website">
      <StatelessService ServiceTypeName="WebsiteType" InstanceCount="[Website_InstanceCount]">
        <SingletonPartition />
      </StatelessService>
    </Service>
  </DefaultServices>
  <Certificates>
    <EndpointCertificate X509FindValue="‎0000000000000" Name="FabricFront" />
  </Certificates>
</ApplicationManifest>

What happens in Local Cluster when launched:

• Engine HTTP on port 1212 always works
• Website HTTPS endpoint on port 443 works ONLY if the Engine HTTPS endpoint is removed as endpoint and from Engine.cs
• Engine HTTPS endpoint NEVER works even if Website is switched to HTTP and is so the only Secure endpoint in the solution, the browser show "ERR_CONNECTION RESET" error.

On ServiceFabric Explorer I see both active and Running, no errors in the Output. I tried other ports with same results.

How can I make this work?

Answer 1

The error was that I was placing <Policies> in ApplicationManifest.xml on the top of the two <ServiceManifestImport>. Each Service needs a different <Policy> inside its own<ServiceManifestImport> just below <ConfigOverrides />.

Answer 2

It´s a bit unclear with your question but if you are trying to run one on port 1601 and one on 443 and only 443 succeeds. Then it might be a privilegie problem? Different ports requires different privilegies.

On the other hand if you are trying to bind both on port 443 then it´s likely you get a conflict since they both use the same port and url. We had the same problem and we managed to get past it by doing the following:

1. Creating a HttpSetup Application that runs a powershell script that

   1. Installs our certificate and registers it using netsh

      &netsh http add sslcert hostnameport="${EndpointHost}:${EndpointPort}" certhash=$CertThumbprint certstorename=$CertStore appid=$AppId

   2. Binds the certificates urls, using netsh:

      &netsh http add urlacl url=$ReservationUrl"

      Example urls

      https://mydnsname.com/

      https://mydnsname.com/api

2. In our main application we then bind our services to the full urls since with full urls there is no conflict. We pass the urls to the cluster via environment parameters.

NOTE: The reason we had to split up into two applications was because our main application deployed continously on each commit. And when netsh was run from multiple deployments at the same time it locked up and hanged on the nodes.

ApplicationManifest.xml

<ServiceManifestImport>
  <ServiceManifestRef ServiceManifestName="ApiPkg" ServiceManifestVersion="1.0.0" />
  <EnvironmentOverrides CodePackageRef="Code">
    <EnvironmentVariable Name="EndpointUri" Value="[Api_EndpointUri]" />
    <EnvironmentVariable Name="CertThumbprint" Value="[Api_CertThumbprint]" />
  </EnvironmentOverrides>
</ServiceManifestImport>
<ServiceManifestImport>
  <ServiceManifestRef ServiceManifestName="UiPkg" ServiceManifestVersion="1.0.0" />
  <EnvironmentOverrides CodePackageRef="Code">
    <EnvironmentVariable Name="EndpointUri" Value="[App_EndpointUri]" />
    <EnvironmentVariable Name="CertThumbprint" Value="[App_CertThumbprint]" />
  </EnvironmentOverrides>
</ServiceManifestImport>

ServiceManifest.xml (for both packages)

<?xml version="1.0" encoding="utf-8"?>
<ServiceManifest Name="UiPkg"
                 Version="1.0.0"
                 xmlns="http://schemas.microsoft.com/2011/01/fabric"
                 xmlns:xsd="http://www.w3.org/2001/XMLSchema"
                 xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
  <ServiceTypes>
    <StatelessServiceType ServiceTypeName="UiType" />
  </ServiceTypes>

  <CodePackage Name="Code" Version="1.0.0">
    <EntryPoint>
      <ExeHost>
        <Program>Ui.exe</Program>
        <WorkingFolder>CodePackage</WorkingFolder>
      </ExeHost>
    </EntryPoint>
    <EnvironmentVariables>
      <EnvironmentVariable Name="EndpointUri" Value="" />
      <EnvironmentVariable Name="CertThumbprint" Value="" />
    </EnvironmentVariables>
  </CodePackage>

  <!-- Config package is the contents of the Config directoy under PackageRoot that contains an
       independently-updateable and versioned set of custom configuration settings for your service. -->
  <ConfigPackage Name="Config" Version="1.0.0" />

  <Resources>
    <Endpoints>
      <!-- To bind to a specific hostname use netsh from a SetupEntyPoint and change Protocol to tcp here to just open the firewall
      -->
      <Endpoint Name="ServiceEndpoint" Protocol="tcp" Port="443" />
    </Endpoints>
  </Resources>
</ServiceManifest>

Program.cs

var listeningAddress = $"{Environment.GetEnvironmentVariable("Api_EndpointUri")}:443/api/";

_webHost = new WebHostBuilder().UseWebListener()
                               .UseContentRoot(Directory.GetCurrentDirectory())
                               .UseStartup<Startup>()
                               .UseUrls(listeningAddress)
                               .Build();