Skip to content

Commit d5b257c

Browse files
AlexMercedCoderAlexMercedCoder
committed
docs
1 parent 76593b7 commit d5b257c

5 files changed

Lines changed: 117 additions & 8 deletions

File tree

‎docs/getting-started/client_configuration.md‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,45 @@ fs.s3a.secret-key=minioadmin
203203
fs.s3a.path.style.access=true
204204
```
205205

206+
## Apache Flink
207+
208+
### SQL Client Configuration
209+
210+
You can create an Iceberg catalog in Flink SQL using the `rest` catalog type.
211+
212+
```sql
213+
CREATE CATALOG pangolin WITH (
214+
'type'='iceberg',
215+
'catalog-type'='rest',
216+
'uri'='http://localhost:8080/v1/analytics',
217+
'token'='<YOUR_JWT_TOKEN>',
218+
'warehouse'='s3://warehouse-bucket/analytics'
219+
);
220+
221+
USE CATALOG pangolin;
222+
```
223+
224+
> [!NOTE]
225+
> Ensure you have the `flink-table-api-java-bridge` and `iceberg-flink-runtime` jars available in your Flink environment.
226+
227+
## Dremio
228+
229+
Dremio supports connecting to Iceberg REST catalogs as a source.
230+
231+
### Configuration Steps
232+
233+
1. **Add Source**: Click "Add Data Source" and select **"Iceberg REST Catalog"**.
234+
2. **General Settings**:
235+
* **Name**: `pangolin_analytics`
236+
* **Endpoint URL**: `http://pangolin:8080/v1/analytics/iceberg` (Ensure reachable from Dremio)
237+
3. **Authentication**:
238+
* If using **No-Auth Mode**, no further auth is needed.
239+
* If using **Token Auth**, you may need to pass the token via the `header.Authorization` property in the "Advanced Options" -> "Connection Properties" if Dremio's UI doesn't explicitly ask for a Bearer token yet.
240+
* *Property*: `header.Authorization`
241+
* *Value*: `Bearer <YOUR_JWT_TOKEN>`
242+
4. **Credential Vending**:
243+
* Enable **"Use vended credentials"** (supported by Pangolin) to let Dremio use the credentials provided by the catalog endpoint for S3/Azure/GCS access.
244+
206245
## Environment Variables
207246

208247
For production deployments, use environment variables:

‎pangolin_ui/src/lib/components/ui/Select.svelte‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
focus:ring-2 focus:border-transparent
4646
disabled:opacity-50 disabled:cursor-not-allowed
4747
transition-colors"
48+
on:change
4849
>
4950
<option value="" disabled selected>{placeholder}</option>
5051
{#each options as option}

‎pangolin_ui/src/lib/stores/auth.ts‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,26 +30,36 @@ function createAuthStore() {
3030
let authEnabled = true;
3131

3232
try {
33+
console.log('Initializing Auth: Fetching App Config...');
3334
const config = await authApi.getAppConfig();
35+
console.log('App Config Response:', config);
3436
authEnabled = config.auth_enabled;
3537
} catch (configError) {
38+
console.warn('App Config failed:', configError);
3639
// If app-config endpoint doesn't exist, try to detect NO_AUTH mode
3740
// by attempting to access a protected endpoint without auth
41+
console.log('Attempting No-Auth Probe (/api/v1/catalogs)...');
3842
try {
3943
const response = await fetch('/api/v1/catalogs', {
4044
method: 'GET',
4145
headers: { 'Content-Type': 'application/json' }
4246
});
47+
console.log('Probe Response:', response.status, response.ok);
4348

4449
// If we get a 200 without auth, we're in NO_AUTH mode
4550
if (response.ok) {
51+
console.log('Probe Succeeded: No Auth Detected');
4652
authEnabled = false;
47-
}
48-
} catch {
53+
} else {
54+
console.log('Probe Failed: Status', response.status);
55+
}
56+
} catch (probeError) {
57+
console.error('Probe Error:', probeError);
4958
// If fetch fails, assume auth is enabled
5059
authEnabled = true;
5160
}
5261
}
62+
console.log('Determined AuthEnabled:', authEnabled);
5363

5464
if (!authEnabled) {
5565
// NO_AUTH mode - set flag
@@ -60,8 +70,10 @@ function createAuthStore() {
6070
const token = localStorage.getItem('auth_token');
6171
const userStr = localStorage.getItem('auth_user');
6272

63-
if (token === 'no-auth-mode' && userStr) {
64-
try {
73+
// Relaxed check: Allow either specific 'no-auth-mode' token OR a real JWT if one was set by a manual login
74+
if (token && userStr) {
75+
try {
76+
console.log('Restoring existing session in No-Auth mode:', token.substring(0, 10) + '...');
6577
const user = JSON.parse(userStr);
6678
update(state => ({
6779
...state,

‎pangolin_ui/src/routes/catalogs/[name]/edit/+page.svelte‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@
4040
// Pre-populate form
4141
warehouseName = catalog.warehouse_name || '';
4242
storageLocation = catalog.storage_location || '';
43+
// Mark as manually set initially to prevent auto-overwriting existing custom paths on load
44+
manualLocation = true;
4345
} catch (error: any) {
4446
notifications.error(`Failed to load catalog: ${error.message}`);
4547
goto('/catalogs');
@@ -66,6 +68,45 @@
6668
return Object.keys(errors).length === 0;
6769
}
6870
71+
let manualLocation = false;
72+
73+
// Helper to determine type from config keys
74+
function getStorageType(config: Record<string, string> | undefined): 's3' | 'azure' | 'gcs' | 's3' {
75+
if (!config) return 's3';
76+
if (config['s3.bucket']) return 's3';
77+
if (config['adls.account-name'] || config['azure.container']) return 'azure';
78+
if (config['gcs.bucket']) return 'gcs';
79+
return 's3';
80+
}
81+
82+
// Reactive update for storage location
83+
$: if (warehouseName && !manualLocation) {
84+
const selected = warehouses.find(w => w.name === warehouseName);
85+
if (selected?.storage_config) {
86+
const config = selected.storage_config;
87+
const bucket = config['s3.bucket']
88+
|| config['adls.container']
89+
|| config['azure.container']
90+
|| config['gcs.bucket'];
91+
92+
const type = getStorageType(config);
93+
94+
// Only auto-update if we have a valid bucket to form a path
95+
if (bucket) {
96+
if (type === 'azure') {
97+
const account = config['adls.account-name'];
98+
if (account) {
99+
storageLocation = `abfss://${bucket}@${account}.dfs.core.windows.net/${catalog?.name || 'catalog'}`;
100+
}
101+
} else if (type === 's3') {
102+
storageLocation = `s3://${bucket}/${catalog?.name || 'catalog'}`;
103+
} else if (type === 'gcs') {
104+
storageLocation = `gs://${bucket}/${catalog?.name || 'catalog'}`;
105+
}
106+
}
107+
}
108+
}
109+
69110
async function handleSubmit() {
70111
if (!validateForm() || !catalogName) return;
71112
@@ -137,13 +178,15 @@
137178
label="Warehouse"
138179
bind:value={warehouseName}
139180
options={warehouseOptions}
181+
on:change={() => manualLocation = false}
140182
placeholder="Select a warehouse..."
141183
helpText="Optional: Link this catalog to a warehouse for credential vending"
142184
/>
143185

144186
<Input
145187
label="Storage Location"
146188
bind:value={storageLocation}
189+
on:input={() => manualLocation = true}
147190
error={errors.storageLocation}
148191
required
149192
placeholder="s3://bucket/path or /local/path"

‎pangolin_ui/src/routes/catalogs/new/+page.svelte‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -86,15 +86,22 @@
8686
}
8787
});
8888
89+
let manualLocation = false;
90+
8991
// Auto-fill storage location logic (only for Local)
90-
$: if (!isFederated && warehouseName && !storageLocation) {
92+
$: if (!isFederated && warehouseName && !manualLocation) {
93+
console.log('Triggering auto-fill. Warehouse:', warehouseName);
9194
const selected = warehouses.find(w => w.value === warehouseName);
95+
console.log('Selected Warehouse Object:', selected);
9296
if (selected?.full?.storage_config) {
9397
const w = selected.full;
9498
const bucket = w.storage_config?.['s3.bucket']
9599
|| w.storage_config?.['adls.container']
96100
|| w.storage_config?.['azure.container']
97-
|| w.storage_config?.['gcs.bucket'];
101+
|| w.storage_config?.['gcs.bucket']
102+
|| w.storage_config?.['bucket']; // fallback
103+
104+
console.log('Found Bucket:', bucket);
98105
99106
const type = w.storage_config?.['s3.bucket'] ? 's3'
100107
: w.storage_config?.['adls.account-name'] ? 'azure'
@@ -109,9 +116,14 @@
109116
} else if (type === 'gcs') {
110117
storageLocation = `gs://${bucket}/${name || 'catalog'}`;
111118
}
112-
}
119+
console.log('Set storageLocation:', storageLocation);
120+
} else {
121+
console.warn('No bucket found in storage_config:', w.storage_config);
122+
}
113123
}
114-
}
124+
} else {
125+
console.log('Skipping auto-fill. IsFederated:', isFederated, 'Warehouse:', warehouseName, 'Manual:', manualLocation);
126+
}
115127
116128
async function handleSubmit() {
117129
if (validationResult && !validationResult.available) {
@@ -248,6 +260,7 @@
248260
label="Warehouse (Optional)"
249261
bind:value={warehouseName}
250262
options={warehouses}
263+
on:change={() => { console.log('Resetting manualLocation'); manualLocation = false; }}
251264
placeholder={loadingWarehouses ? "Loading warehouses..." : "Select a warehouse (Optional)"}
252265
disabled={loading || loadingWarehouses}
253266
/>
@@ -263,6 +276,7 @@
263276
<Input
264277
label="Storage Location (Optional)"
265278
bind:value={storageLocation}
279+
on:input={() => manualLocation = true}
266280
placeholder="s3://bucket/path/to/catalog"
267281
disabled={loading}
268282
/>

0 commit comments

Comments
 (0)