OPC # 0001: Extract OPC into standalone repo

This commit is contained in:
amadzarak
2026-04-25 17:26:42 -04:00
commit 42383bdc03
170 changed files with 21365 additions and 0 deletions
@@ -0,0 +1,358 @@
using ControlPlane.Core.Config;
using ControlPlane.Core.Messages;
using Docker.DotNet;
using Docker.DotNet.Models;
using MassTransit;
using Microsoft.Extensions.Options;
namespace ControlPlane.Worker.Services;
/// <summary>
/// Manages Clarity.Server Docker containers for provisioned tenants.
/// Container naming convention: {env}-app-clarity-{siteCode}
/// e.g. fdev-app-clarity-01000014
/// </summary>
public class ClarityContainerService(
IConfiguration config,
IOptions<ClarityInfraOptions> infraOptions,
IPublishEndpoint bus,
ILogger<ClarityContainerService> logger)
{
private ClarityInfraOptions Infra => infraOptions.Value;
// The image to run - override via config for prod registries
private string ImageName => config["Docker:ClarityServerImage"] ?? "clarity-server:latest";
private DockerClient CreateClient()
{
var uri = config["Docker:Socket"] ?? "npipe://./pipe/docker_engine";
return new DockerClientConfiguration(new Uri(uri)).CreateClient();
}
/// <summary>
/// Derives the container name from environment + siteCode.
/// Convention: {env}-app-clarity-{siteCode}
/// </summary>
public static string ContainerName(string environment, string siteCode) =>
$"{environment.ToLowerInvariant()}-app-clarity-{siteCode.ToLowerInvariant()}";
/// <summary>
/// Pulls the image (if not present locally), starts the container on the managed network,
/// and writes an nginx conf.d snippet so traffic routes in.
/// No host port binding — nginx reaches the container via Docker DNS on the shared network.
/// </summary>
public async Task<string> StartTenantContainerAsync(
string environment,
string siteCode,
string subdomain,
string keycloakRealm,
string? postgresConnectionString,
string? vaultToken,
Guid jobId,
CancellationToken cancellationToken)
{
using var docker = CreateClient();
var name = ContainerName(environment, siteCode);
// Stop and remove any existing container with this name (idempotent reprovision)
await TryRemoveExistingAsync(docker, name, cancellationToken);
// Pull image if not already local
await EnsureImageAsync(docker, cancellationToken);
// All service URLs use stable Docker DNS names on the managed network — no host ports involved.
var container = await docker.Containers.CreateContainerAsync(new CreateContainerParameters
{
Name = name,
Image = ImageName,
Env =
[
"ASPNETCORE_ENVIRONMENT=Production",
"ASPNETCORE_URLS=http://+:8080",
$"TenantSubdomain={subdomain}",
$"Keycloak__BaseUrl={Infra.KeycloakPublicUrl}",
$"Keycloak__InternalUrl={Infra.KeycloakInternalUrl}",
$"Keycloak__Realm={keycloakRealm}",
$"Vault__Address={Infra.VaultInternalUrl}",
.. (vaultToken is not null
? (string[])[$"Vault__Token={vaultToken}"]
: []),
.. (postgresConnectionString is not null
? (string[])[$"ConnectionStrings__postgresdb={postgresConnectionString}"]
: []),
],
HostConfig = new HostConfig
{
NetworkMode = Infra.Network,
RestartPolicy = new RestartPolicy { Name = RestartPolicyKind.UnlessStopped },
},
Labels = new Dictionary<string, string>
{
["clarity.managed"] = "true",
["clarity.subdomain"] = subdomain,
["clarity.siteCode"] = siteCode,
["clarity.env"] = environment,
},
}, cancellationToken);
// Ensure Keycloak and Vault are reachable on the managed network via their Docker DNS aliases.
// Aspire places them on its own bridge; tenant containers on clarity-net need them aliased here.
await EnsureContainerOnNetworkAsync(docker, "keycloak", Infra.Network, "keycloak", cancellationToken);
await EnsureContainerOnNetworkAsync(docker, "vault", Infra.Network, "vault", cancellationToken);
var started = await docker.Containers.StartContainerAsync(container.ID, null, cancellationToken);
if (!started)
throw new InvalidOperationException($"Docker failed to start container {name} (id={container.ID}).");
logger.LogInformation("Started container {Name} on {Network} (image: {Image})", name, Infra.Network, ImageName);
await WriteNginxConfigAsync(subdomain, name, jobId, cancellationToken);
return name;
}
/// <summary>
/// Stops and removes a tenant container. Called from InfrastructureStep.CompensateAsync.
/// </summary>
public async Task StopAndRemoveAsync(string containerName, CancellationToken cancellationToken)
{
using var docker = CreateClient();
await TryRemoveExistingAsync(docker, containerName, cancellationToken);
logger.LogInformation("Removed container {Name}", containerName);
}
// -- helpers --
private async Task EnsureImageAsync(DockerClient docker, CancellationToken cancellationToken)
{
var images = await docker.Images.ListImagesAsync(new ImagesListParameters
{
Filters = new Dictionary<string, IDictionary<string, bool>>
{
["reference"] = new Dictionary<string, bool> { [ImageName] = true }
}
}, cancellationToken);
if (images.Count > 0)
{
logger.LogInformation("Image {Image} already present locally.", ImageName);
return;
}
// Local image (no registry host) — pulling from Docker Hub will always fail.
// The image must be built manually before provisioning.
var isLocalOnly = !ImageName.Contains('/') || ImageName.StartsWith("localhost/");
if (isLocalOnly)
{
throw new InvalidOperationException(
$"Image '{ImageName}' was not found locally and cannot be pulled from a registry. " +
$"Build it first from the repo root:{Environment.NewLine}" +
$" docker build -f Clarity.Server/Dockerfile -t {ImageName} ." +
$"{Environment.NewLine}Then retry provisioning.");
}
// Registry image — attempt pull
logger.LogInformation("Pulling image {Image} from registry...", ImageName);
var (repo, tag) = SplitImageTag(ImageName);
await docker.Images.CreateImageAsync(
new ImagesCreateParameters { FromImage = repo, Tag = tag },
null,
new Progress<JSONMessage>(m =>
{
if (!string.IsNullOrWhiteSpace(m.Status))
logger.LogDebug("[docker pull] {Status} {Progress}", m.Status, m.ProgressMessage);
}),
cancellationToken);
}
// -- nginx conf.d helpers --
/// <summary>
/// Writes /NginxConfig/conf.d/{subdomain}.conf so nginx routes
/// {subdomain}.clarity.test → the containe
/// Then signals nginx to reload its config without dropping connections.
/// </summary>
private async Task WriteNginxConfigAsync(string subdomain, string containerName, Guid jobId, CancellationToken ct)
{
var confDPath = config["Nginx:ConfDPath"];
if (string.IsNullOrWhiteSpace(confDPath))
{
logger.LogWarning("Nginx:ConfDPath is not configured — skipping nginx conf write for {Subdomain}.", subdomain);
return;
}
var confContent = $$$"""
# Auto-generated by ControlPlane.Worker — do not edit manually.
# Tenant: {{{subdomain}}}
server {
listen 443 ssl;
server_name {{{subdomain}}}.{{{Infra.Domain}}};
ssl_certificate {{{Infra.NginxCertPath}}};
ssl_certificate_key {{{Infra.NginxCertKeyPath}}};
location / {
# Docker DNS resolves the container name on the managed network
set $upstream http://{{{containerName}}}:8080;
proxy_pass $upstream;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
}
""";
var confFile = Path.Combine(confDPath, $"{subdomain}.conf");
await File.WriteAllTextAsync(confFile, confContent, ct);
logger.LogInformation("Wrote nginx config for {Subdomain} → {Container}", subdomain, containerName);
await ReloadNginxAsync(jobId, subdomain, ct);
}
public async Task RemoveNginxConfigAsync(string subdomain, CancellationToken ct)
{
var confDPath = config["Nginx:ConfDPath"];
if (string.IsNullOrWhiteSpace(confDPath)) return;
var confFile = Path.Combine(confDPath, $"{subdomain}.conf");
if (File.Exists(confFile))
{
File.Delete(confFile);
logger.LogInformation("Removed nginx config for {Subdomain}", subdomain);
await ReloadNginxAsync(Guid.Empty, subdomain, ct);
}
}
/// Sends SIGHUP to the nginx container which triggers a graceful config reload.
private async Task ReloadNginxAsync(Guid jobId, string subdomain, CancellationToken ct)
{
try
{
using var docker = CreateClient();
// Find the nginx container by image name — Aspire appends a random suffix to the name
// so we can't rely on the static name "nginx".
var containers = await docker.Containers.ListContainersAsync(
new ContainersListParameters
{
Filters = new Dictionary<string, IDictionary<string, bool>>
{
["ancestor"] = new Dictionary<string, bool> { ["nginx"] = true }
}
}, ct);
var nginx = containers.FirstOrDefault();
if (nginx is null)
{
logger.LogWarning("nginx container not found — skipping reload.");
return;
}
await docker.Containers.KillContainerAsync(nginx.ID, new ContainerKillParameters { Signal = "HUP" }, ct);
var containerName = nginx.Names.FirstOrDefault() ?? nginx.ID;
logger.LogInformation("nginx reloaded (container: {Name}).", containerName);
if (jobId != Guid.Empty)
{
await bus.Publish(new ProvisioningProgressEvent
{
JobId = jobId,
Type = "nginx_reloaded",
Step = "Container Launch",
Message = $"nginx reloaded — route for {subdomain}.{Infra.Domain} is live.",
}, ct);
}
}
catch (Exception ex)
{
logger.LogWarning(ex, "Failed to reload nginx — new tenant route may not be active until next nginx restart.");
if (jobId != Guid.Empty)
{
await bus.Publish(new ProvisioningProgressEvent
{
JobId = jobId,
Type = "diagnostic",
Step = "Container Launch",
Message = "nginx reload failed — route may not be active.",
Detail = ex.ToString(),
}, ct);
}
}
}
// -- docker helpers --
private static async Task TryRemoveExistingAsync(DockerClient docker, string name, CancellationToken cancellationToken)
{
try
{
await docker.Containers.StopContainerAsync(name,
new ContainerStopParameters { WaitBeforeKillSeconds = 5 }, cancellationToken);
await docker.Containers.RemoveContainerAsync(name,
new ContainerRemoveParameters { Force = true }, cancellationToken);
}
catch (DockerContainerNotFoundException) { /* already gone - fine */ }
catch (DockerApiException ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound) { /* same */ }
}
private static (string repo, string tag) SplitImageTag(string image)
{
var colon = image.LastIndexOf(':');
return colon < 0 ? (image, "latest") : (image[..colon], image[(colon + 1)..]);
}
/// <summary>
/// Connects <paramref name="containerName"/> to <paramref name="network"/> with the given
/// <paramref name="alias"/> if it isn't already connected.
/// Silently no-ops if the container isn't found (it may not be running in all environments).
/// </summary>
private async Task EnsureContainerOnNetworkAsync(
DockerClient docker,
string containerName,
string network,
string alias,
CancellationToken cancellationToken)
{
try
{
var inspect = await docker.Containers.InspectContainerAsync(containerName, cancellationToken);
if (inspect.NetworkSettings.Networks.TryGetValue(network, out var existing))
{
// Already connected — check whether our alias is present.
var hasAlias = existing.Aliases?.Contains(alias, StringComparer.OrdinalIgnoreCase) == true;
if (hasAlias) return;
// Connected but without the alias — disconnect so we can reconnect with it.
await docker.Networks.DisconnectNetworkAsync(network, new NetworkDisconnectParameters
{
Container = inspect.ID,
Force = true,
}, cancellationToken);
}
await docker.Networks.ConnectNetworkAsync(network, new NetworkConnectParameters
{
Container = inspect.ID,
EndpointConfig = new EndpointSettings
{
Aliases = [alias],
},
}, cancellationToken);
logger.LogInformation("Connected container '{Container}' to network '{Network}' with alias '{Alias}'.", containerName, network, alias);
}
catch (DockerContainerNotFoundException)
{
logger.LogWarning("Container '{Container}' not found — skipping network connect.", containerName);
}
catch (DockerApiException ex) when (ex.StatusCode == System.Net.HttpStatusCode.NotFound)
{
logger.LogWarning("Container '{Container}' not found — skipping network connect.", containerName);
}
catch (Exception ex)
{
logger.LogWarning(ex, "Could not connect '{Container}' to '{Network}' — tenant JWT validation may fail.", containerName, network);
}
}
}
@@ -0,0 +1,194 @@
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
namespace ControlPlane.Worker.Services;
public class KeycloakAdminClient
{
private readonly HttpClient _http;
private readonly string _baseUrl;
private readonly string _adminUser;
private readonly string _adminPassword;
private readonly ILogger<KeycloakAdminClient> _logger;
private static readonly JsonSerializerOptions JsonOpts = new(JsonSerializerDefaults.Web);
public KeycloakAdminClient(IConfiguration config, ILogger<KeycloakAdminClient> logger)
{
_logger = logger;
_baseUrl = (config["Keycloak:AuthServerUrl"] ?? config["Keycloak:BaseUrl"])?.TrimEnd('/') ?? "http://localhost:8080";
_adminUser = config["Keycloak:AdminUser"] ?? "admin";
_adminPassword = config["Keycloak:AdminPassword"] ?? "admin";
var maskedPw = _adminPassword.Length > 2 ? $"{_adminPassword[0]}***{_adminPassword[^1]}" : "***";
_logger.LogInformation("KeycloakAdminClient base URL: {Url}, user: {User}, password: {Password}",
_baseUrl, _adminUser, maskedPw);
_http = new HttpClient { BaseAddress = new Uri(_baseUrl) };
}
private async Task AuthorizeAsync(CancellationToken ct)
{
var form = new FormUrlEncodedContent(new Dictionary<string, string>
{
["grant_type"] = "password",
["client_id"] = "admin-cli",
["username"] = _adminUser,
["password"] = _adminPassword,
});
var res = await _http.PostAsync("/realms/master/protocol/openid-connect/token", form, ct);
res.EnsureSuccessStatusCode();
using var doc = JsonDocument.Parse(await res.Content.ReadAsStringAsync(ct));
var token = doc.RootElement.GetProperty("access_token").GetString()!;
_http.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token);
}
public async Task CreateRealmAsync(string realmId, string displayName, CancellationToken ct)
{
await AuthorizeAsync(ct);
var res = await _http.PostAsync("/admin/realms", Json(new
{
realm = realmId,
displayName = displayName,
enabled = true,
registrationAllowed = true,
registrationEmailAsUsername = false,
loginWithEmailAllowed = true,
resetPasswordAllowed = true,
verifyEmail = false,
sslRequired = "external",
}), ct);
if (res.StatusCode == System.Net.HttpStatusCode.Conflict)
{
_logger.LogWarning("Realm {Realm} already exists - skipping.", realmId);
return;
}
res.EnsureSuccessStatusCode();
_logger.LogInformation("Realm {Realm} created.", realmId);
}
public async Task DeleteRealmAsync(string realmId, CancellationToken ct)
{
await AuthorizeAsync(ct);
var res = await _http.DeleteAsync($"/admin/realms/{realmId}", ct);
if (res.StatusCode != System.Net.HttpStatusCode.NotFound)
res.EnsureSuccessStatusCode();
_logger.LogInformation("Realm {Realm} deleted.", realmId);
}
public async Task CreateRealmRoleAsync(string realmId, string roleName, string description, CancellationToken ct)
{
await AuthorizeAsync(ct);
var res = await _http.PostAsync($"/admin/realms/{realmId}/roles",
Json(new { name = roleName, description }), ct);
if (res.StatusCode != System.Net.HttpStatusCode.Conflict)
res.EnsureSuccessStatusCode();
}
public async Task<string> CreateUserAsync(string realmId, string email, string firstName, CancellationToken ct)
{
await AuthorizeAsync(ct);
var res = await _http.PostAsync($"/admin/realms/{realmId}/users",
Json(new { username = email, email, firstName, enabled = true, emailVerified = true }), ct);
if (res.StatusCode != System.Net.HttpStatusCode.Conflict)
res.EnsureSuccessStatusCode();
return await GetUserIdAsync(realmId, email, ct);
}
public async Task<string> GetUserIdAsync(string realmId, string email, CancellationToken ct)
{
await AuthorizeAsync(ct);
var res = await _http.GetAsync(
$"/admin/realms/{realmId}/users?email={Uri.EscapeDataString(email)}&exact=true", ct);
res.EnsureSuccessStatusCode();
using var doc = JsonDocument.Parse(await res.Content.ReadAsStringAsync(ct));
var users = doc.RootElement.EnumerateArray().ToList();
if (users.Count == 0)
throw new InvalidOperationException($"User {email} not found in realm {realmId}.");
return users[0].GetProperty("id").GetString()!;
}
public async Task AssignRealmRoleAsync(string realmId, string userId, string roleName, CancellationToken ct)
{
await AuthorizeAsync(ct);
var roleRes = await _http.GetAsync($"/admin/realms/{realmId}/roles/{roleName}", ct);
roleRes.EnsureSuccessStatusCode();
var roleJson = await roleRes.Content.ReadAsStringAsync(ct);
var res = await _http.PostAsync(
$"/admin/realms/{realmId}/users/{userId}/role-mappings/realm",
new StringContent($"[{roleJson}]", Encoding.UTF8, "application/json"), ct);
res.EnsureSuccessStatusCode();
}
public async Task CreateClientAsync(string realmId, object clientRepresentation, CancellationToken ct)
{
await AuthorizeAsync(ct);
var res = await _http.PostAsync($"/admin/realms/{realmId}/clients",
Json(clientRepresentation), ct);
if (res.StatusCode != System.Net.HttpStatusCode.Conflict)
res.EnsureSuccessStatusCode();
}
/// <summary>
/// Returns the internal Keycloak UUID for a client by its clientId string.
/// </summary>
public async Task<string> GetClientUuidAsync(string realmId, string clientId, CancellationToken ct)
{
await AuthorizeAsync(ct);
var res = await _http.GetAsync(
$"/admin/realms/{realmId}/clients?clientId={Uri.EscapeDataString(clientId)}&search=false", ct);
res.EnsureSuccessStatusCode();
using var doc = JsonDocument.Parse(await res.Content.ReadAsStringAsync(ct));
var clients = doc.RootElement.EnumerateArray().ToList();
if (clients.Count == 0)
throw new InvalidOperationException($"Client '{clientId}' not found in realm '{realmId}'.");
return clients[0].GetProperty("id").GetString()!;
}
/// <summary>
/// Adds an audience protocol mapper to a client so that the named audience is included in every
/// access token issued by that client.
/// </summary>
public async Task AddAudienceMapperAsync(string realmId, string clientUuid, string audienceName, CancellationToken ct)
{
await AuthorizeAsync(ct);
var res = await _http.PostAsync(
$"/admin/realms/{realmId}/clients/{clientUuid}/protocol-mappers/models",
Json(new
{
name = $"audience-{audienceName}",
protocol = "openid-connect",
protocolMapper = "oidc-audience-mapper",
consentRequired = false,
config = new Dictionary<string, string>
{
["included.client.audience"] = audienceName,
["id.token.claim"] = "false",
["access.token.claim"] = "true",
},
}), ct);
if (res.StatusCode != System.Net.HttpStatusCode.Conflict)
res.EnsureSuccessStatusCode();
_logger.LogInformation("Added audience mapper '{Audience}' to client {ClientUuid} in realm {Realm}.",
audienceName, clientUuid, realmId);
}
public async Task SendRequiredActionsEmailAsync(
string realmId, string userId, IEnumerable<string> actions, CancellationToken ct)
{
await AuthorizeAsync(ct);
var res = await _http.PutAsync(
$"/admin/realms/{realmId}/users/{userId}/execute-actions-email?lifespan=86400",
new StringContent(JsonSerializer.Serialize(actions, JsonOpts), Encoding.UTF8, "application/json"),
ct);
res.EnsureSuccessStatusCode();
}
private static StringContent Json(object payload) =>
new(JsonSerializer.Serialize(payload, JsonOpts), Encoding.UTF8, "application/json");
}