{
 "name": "Platform facts",
 "edition": "2026-11-10",
 "read_date": "2026-11-10",
 "source": "https://datalakehousecompare.com/datasets/",
 "license": "CC BY 4.0",
 "cite": "Lakehouse Compare, \"Lakehouse Compute Bill\", edition 10 Nov 2026, https://datalakehousecompare.com/datasets/ (CC BY 4.0).",
 "description": "Every published fact behind the platform profiles, with its topic, source URL and read date. Alphabetical by platform.",
 "rows": [
  {
   "platform": "Amazon EMR plus Amazon Athena on Apache Iceberg",
   "topic": "rewrite",
   "fact": "The EMR runtime for Spark is described as 100% API-compatible with open-source Spark, so existing Spark jobs need no rewrite; EMR is native Spark on EC2, EKS or Serverless.",
   "source_type": "cloud-provider-docs",
   "source_url": "https://aws.amazon.com/emr/features/spark/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Amazon EMR plus Amazon Athena on Apache Iceberg",
   "topic": "speed",
   "fact": "AWS states its runtime executes up to 4.5x faster than open-source Spark and gives 2.7x faster Iceberg write performance; the page gives no workload, scale or configuration.",
   "source_type": "vendor",
   "source_url": "https://aws.amazon.com/emr/features/spark/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Amazon EMR plus Amazon Athena on Apache Iceberg",
   "topic": "stack",
   "fact": "EMR 7.x release table lists Spark 3.5.x builds (up to 3.5.8-amzn-0); the EMR Spark feature page says EMR supports Apache Iceberg v3 and Spark 4.0.",
   "source_type": "cloud-provider-docs",
   "source_url": "https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-release-app-versions-7.x.html",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Amazon EMR plus Amazon Athena on Apache Iceberg",
   "topic": "tune",
   "fact": "EMR enables adaptive query execution, dynamic partition pruning, bloom filter join and optimized join reorder; docs say AQE adaptive join conversion is on by default and can be switched off with a Spark setting.",
   "source_type": "cloud-provider-docs",
   "source_url": "https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-spark-performance.html",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Amazon EMR plus Amazon Athena on Apache Iceberg",
   "topic": "cost",
   "fact": "EMR on EC2 adds a per-instance uplift in addition to EC2: the AWS price list shows $0.048 per hour for EMR m5.xlarge in us-east-1 (publication 2026-11-10).",
   "source_type": "cloud-provider-docs",
   "source_url": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/ElasticMapReduce/current/index.json",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Amazon EMR plus Amazon Athena on Apache Iceberg",
   "topic": "cost",
   "fact": "EMR on EKS uplift in us-east-1 is $0.007084 per vCPU-hour and $0.000777875 per GB-hour, in addition to the EC2 or Fargate cost.",
   "source_type": "cloud-provider-docs",
   "source_url": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/ElasticMapReduce/current/index.json",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Amazon EMR plus Amazon Athena on Apache Iceberg",
   "topic": "cost",
   "fact": "The EMR pricing page worked example for EMR Serverless uses $0.052624 per vCPU-hour and $0.0057785 per GB-hour; a job with 100 vCPU for 30 minutes plus 200 more vCPU for 15 minutes costs $9.596275 in that example.",
   "source_type": "cloud-provider-docs",
   "source_url": "https://aws.amazon.com/emr/pricing/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Amazon EMR plus Amazon Athena on Apache Iceberg",
   "topic": "cost",
   "fact": "Athena SQL costs $5.00 per TB scanned in us-east-1; Athena for Apache Spark costs $0.35 per DPU-hour; provisioned capacity costs $0.30 per DPU-hour.",
   "source_type": "cloud-provider-docs",
   "source_url": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonAthena/current/index.json",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Amazon EMR plus Amazon Athena on Apache Iceberg",
   "topic": "rewrite",
   "fact": "Athena for Apache Spark (version 3.5) accepts Spark code from SageMaker Unified Studio notebooks or any Spark Connect compatible client; PySpark engine version 3 offers the console notebook experience.",
   "source_type": "cloud-provider-docs",
   "source_url": "https://docs.aws.amazon.com/athena/latest/ug/notebooks-spark.html",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Amazon EMR plus Amazon Athena on Apache Iceberg",
   "topic": "stack",
   "fact": "Athena supports Apache Iceberg 1.4.2 for read, time travel, write and DDL, only with tables in the AWS Glue Data Catalog and Glue optimistic locking; DDL is not supported on tables registered with Lake Formation.",
   "source_type": "cloud-provider-docs",
   "source_url": "https://docs.aws.amazon.com/athena/latest/ug/querying-iceberg.html",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Amazon EMR plus Amazon Athena on Apache Iceberg",
   "topic": "tune",
   "fact": "Athena compaction is manual: run OPTIMIZE ... REWRITE DATA USING BIN_PACK and VACUUM yourself on Iceberg tables.",
   "source_type": "cloud-provider-docs",
   "source_url": "https://docs.aws.amazon.com/athena/latest/ug/querying-iceberg-data-optimization.html",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Amazon EMR plus Amazon Athena on Apache Iceberg",
   "topic": "tune",
   "fact": "Glue Data Catalog Iceberg compaction is disabled by default for new tables and must be enabled per table with an IAM role.",
   "source_type": "cloud-provider-docs",
   "source_url": "https://docs.aws.amazon.com/glue/latest/dg/enable-compaction.html",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Amazon EMR plus Amazon Athena on Apache Iceberg",
   "topic": "cost",
   "fact": "Glue Iceberg compaction costs $0.44 per DPU-hour in us-east-1; Glue 6.0+ ETL jobs cost $0.308 per DPU-hour and older Glue ETL $0.44.",
   "source_type": "cloud-provider-docs",
   "source_url": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AWSGlue/current/index.json",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Amazon EMR plus Amazon Athena on Apache Iceberg",
   "topic": "tune",
   "fact": "S3 Tables compaction is on by default for all tables with a 512 MB default target file size (configurable 64 to 512 MB) and also applies row-level deletes.",
   "source_type": "cloud-provider-docs",
   "source_url": "https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-tables-maintenance.html",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Amazon EMR plus Amazon Athena on Apache Iceberg",
   "topic": "cost",
   "fact": "S3 Tables in us-east-1: compaction is $0.005 per GB processed plus $0.002 per 1,000 objects compacted; sort or Z-order compaction is $0.01 per GB plus $0.002 per 1,000 objects; storage is $0.0265 per GB-month for the first 50 TB.",
   "source_type": "cloud-provider-docs",
   "source_url": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonS3/current/index.json",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Amazon EMR plus Amazon Athena on Apache Iceberg",
   "topic": "where",
   "fact": "All of these are AWS services, so they run only on AWS; EMR, Athena, Glue and S3 are priced in us-east-1 on the AWS price list.",
   "source_type": "cloud-provider-docs",
   "source_url": "https://aws.amazon.com/emr/features/spark/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Databricks Data Intelligence Platform on AWS",
   "topic": "rewrite",
   "fact": "Photon is a native C++ vectorized engine that keeps Spark's Catalyst planner and works with existing Spark API code; unsupported operations fall back to the Spark runtime.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.databricks.com/aws/en/compute/photon",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Databricks Data Intelligence Platform on AWS",
   "topic": "tune",
   "fact": "Photon is on by default for classic all-purpose compute, jobs compute and classic pipelines; API users must set runtime_engine to PHOTON. Photon instance types use DBUs at a different rate than the same instance without Photon.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.databricks.com/aws/en/compute/photon",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Databricks Data Intelligence Platform on AWS",
   "topic": "speed",
   "fact": "Docs claim Photon gives up to 5x better price/performance for data and analytics workloads versus other cloud data warehouses; no method is given on this page.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.databricks.com/aws/en/compute/photon",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Databricks Data Intelligence Platform on AWS",
   "topic": "rewrite",
   "fact": "Serverless compute for notebooks and jobs supports only Spark Connect APIs; Spark RDD APIs, R, Maven coordinates, global temp views, DBFS mounts with instance profiles and Databricks Container Services are not supported. ANSI mode is on by default and the default time zone is UTC, which can change results of existing jobs.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.databricks.com/aws/en/compute/serverless/limitations",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Databricks Data Intelligence Platform on AWS",
   "topic": "tune",
   "fact": "On serverless compute Databricks allocates and manages compute so users do not provision clusters; the Spark UI and Spark logs are not available, query profile and client-side logs are used instead.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.databricks.com/aws/en/compute/serverless/limitations",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Databricks Data Intelligence Platform on AWS",
   "topic": "tune",
   "fact": "Serverless compute runs workloads without provisioning compute in your cloud account, and Databricks says this reduces the need to manage compute resources. It requires Unity Catalog to reach external data sources.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.databricks.com/aws/en/compute/serverless/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Databricks Data Intelligence Platform on AWS",
   "topic": "stack",
   "fact": "Databricks Runtime 18 LTS ships Apache Spark 4.1.0 (released June 10, 2026); Runtime 17.3 LTS ships Spark 4.0.0; Runtime 16.4 LTS and 15.4 LTS ship Spark 3.5.x. Runtime 19 ships Spark 4.2.0 (July 23, 2026).",
   "source_type": "vendor-docs",
   "source_url": "https://docs.databricks.com/aws/en/release-notes/runtime/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Databricks Data Intelligence Platform on AWS",
   "topic": "stack",
   "fact": "Delta Lake tables can generate Iceberg metadata (UniForm, Iceberg reads) from Databricks Runtime 14.3 LTS so Iceberg clients read Delta data without rewriting files.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.databricks.com/aws/en/delta/uniform",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Databricks Data Intelligence Platform on AWS",
   "topic": "where",
   "fact": "Databricks on AWS lists many AWS regions including us-east-1 (US East, Northern Virginia), us-east-2, us-west-1, eu-west-1, eu-central-1, ap-southeast-2 and us-gov-west-1; the platform is also hosted on Azure and Google Cloud. Some features are only in a subset of regions.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.databricks.com/aws/en/resources/supported-regions",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Databricks Data Intelligence Platform on AWS",
   "topic": "speed",
   "fact": "Databricks announced an audited 100TB TPC-DS result of 32,941,245 QphDS for Databricks SQL, 2.2x the previous record held by Alibaba, with total system cost 10% lower on list pricing. This is a SQL warehouse result, not a Spark batch ETL result.",
   "source_type": "vendor",
   "source_url": "https://www.databricks.com/blog/2021/11/02/databricks-sets-official-data-warehousing-performance-record.html",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Databricks Data Intelligence Platform on AWS",
   "topic": "speed",
   "fact": "The same Databricks blog says research from Barcelona Supercomputing Center found Databricks 2.7x faster and 12x better in price-performance than Snowflake on a TPC-DS derivative. This is a vendor-reported claim of a third-party study; the study itself was not fetched.",
   "source_type": "vendor",
   "source_url": "https://www.databricks.com/blog/2021/11/02/databricks-sets-official-data-warehousing-performance-record.html",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Databricks Data Intelligence Platform on AWS",
   "topic": "cost",
   "fact": "Databricks bills compute in DBUs at per-second granularity with pay-as-you-go or committed-use discounts, and no up-front costs. The pricing pages render their rates by script; the AWS pricing data file those pages load lists Jobs Compute at $0.15 per DBU on the plan it names Premium (r6i.4xlarge: 4.08 DBU per hour). The file names no region.",
   "source_type": "vendor",
   "source_url": "https://www.databricks.com/product/aws-pricing",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Databricks Data Intelligence Platform on AWS",
   "topic": "cost",
   "fact": "Jobs pricing offers serverless (Performance Optimized and Standard modes) and classic clusters; the jobs pricing page maps plan names across clouds (its Premium tier on Azure corresponds to Enterprise on AWS and GCP), while the AWS pricing data file lists the $0.15 Jobs Compute rate under a plan it names Premium. Photon instances consume DBUs at a different rate.",
   "source_type": "vendor",
   "source_url": "https://www.databricks.com/product/pricing/jobs",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Dremio",
   "topic": "profile",
   "fact": "SAP and Dremio announced on May 4, 2026 that SAP agreed to acquire Dremio. Terms were not disclosed and the deal was pending regulatory approval at announcement. SAP said Business Data Cloud will become an Apache Iceberg-native lakehouse using Dremio.",
   "source_type": "vendor",
   "source_url": "https://news.sap.com/2026/05/sap-to-acquire-dremio/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Dremio",
   "topic": "profile",
   "fact": "SAP announced on July 6, 2026 that it completed the acquisition of Dremio.",
   "source_type": "vendor",
   "source_url": "https://news.sap.com/2026/07/sap-completes-dremio-acquisition/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Dremio",
   "topic": "profile",
   "fact": "The dremio-oss GitHub repository is licensed Apache-2.0 and its last push was 2025-09-26, so the open-source repository has been quiet while the commercial products (Dremio Cloud, Dremio Enterprise, Dremio Community Edition) continue.",
   "source_type": "open-source-project",
   "source_url": "https://api.github.com/repos/dremio/dremio-oss",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Dremio",
   "topic": "cost",
   "fact": "Dremio Cloud is priced at $0.20 per DCU (Dremio Compute Unit). Dremio Enterprise (self-managed) pricing is 'contact sales'. The free trial for Dremio Cloud is a $400 credit for 30 days.",
   "source_type": "vendor",
   "source_url": "https://www.dremio.com/pricing",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Dremio",
   "topic": "cost",
   "fact": "DCUs are consumed only while engines run. Usage equals engine and replica uptime in seconds (60-second minimum) times the DCU rate for the engine size. Published rates per hour: 2XSmall 16, XSmall 32, Small 64, Medium 128, Large 256, XLarge 512, 2XLarge 1024, 3XLarge 2048. Storage billed in TB-months only for Dremio-hosted storage; LLM tokens billed separately.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.dremio.com/dremio-cloud/admin/engines/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Dremio",
   "topic": "cost",
   "fact": "Billable usage is DCUs (compute), LLM tokens (AI features) and storage. Compute is billed in DCUs only while engines are running.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.dremio.com/dremio-cloud/admin/subscription/usage",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Dremio",
   "topic": "rewrite",
   "fact": "Dremio runs queries on its own columnar execution engine built on Apache Arrow, not on Spark. Docs describe Spark and Flink only as external engines that can use the Open Catalog (Iceberg REST) alongside Dremio. The pages read describe no mode that runs existing Spark jobs, so Spark ETL would stay on Spark or be rewritten as Dremio SQL.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.dremio.com/current/what-is-dremio/architecture",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Dremio",
   "topic": "stack",
   "fact": "The Dremio Cloud data formats page lists Apache Iceberg and Delta Lake as table formats (plus Parquet and text formats). Hudi is not listed on that page. The Open Catalog is built on Apache Polaris and supports the Iceberg REST Catalog API; the docs list AWS Glue Data Catalog as a connectable catalog.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.dremio.com/dremio-cloud/developer/data-formats/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Dremio",
   "topic": "tune",
   "fact": "For Iceberg tables in the Open Catalog, Dremio runs automatic optimization on a dedicated engine it configures: data file compaction, row-level delete handling (position deletes on v2, deletion vectors on v3), clustering, partition evolution and manifest rewriting. The same operations can be run by hand with OPTIMIZE TABLE. Optimization works in batches sized to engine resources.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.dremio.com/dremio-cloud/manage-govern/optimization",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Dremio",
   "topic": "tune",
   "fact": "Engines have settings the admin chooses: size (executors per replica), min and max replicas, last-replica auto-stop (default 2 hours), enqueued time limit (default 5 minutes) and query runtime limit. Default concurrency per replica ranges from 2 (2XSmall) to 20 (3XLarge).",
   "source_type": "vendor-docs",
   "source_url": "https://docs.dremio.com/dremio-cloud/admin/engines/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Dremio",
   "topic": "where",
   "fact": "Dremio Cloud runs on AWS (Azure listed as coming soon). Supported regions: us-east-1, us-east-2, us-west-2, eu-west-1 and eu-central-1. Dremio Enterprise can be deployed on Kubernetes, on premises, or in AWS, Azure or GCP.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.dremio.com/dremio-cloud/about/regions",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Dremio",
   "topic": "speed",
   "fact": "Dremio's founder reported a full 1 TB TPC-DS run (all 99 queries) in 22 seconds on an 8-node m7gd.4xlarge cluster, with no caching, and attributes it to Autonomous Reflections and automatic Iceberg clustering. This is a BI query benchmark run by the vendor, not a Spark batch ETL or compaction test.",
   "source_type": "vendor",
   "source_url": "https://www.dremio.com/blog/breakthrough-announcement-dremio-is-the-fastest-lakehouse-20x-faster-on-tpc-ds/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Dremio",
   "topic": "open",
   "fact": "Dremio published a 2020 TPC-DS methodology post (SF1000 and SF10000, Parquet on S3, Apache JMeter test plans, 58 of 99 queries that ran unmodified on Dremio and Presto distributions). The 2025 22-second TPC-DS post pages read give cluster type but no scripts, scale-factor files or configuration repository.",
   "source_type": "vendor",
   "source_url": "https://www.dremio.com/blog/dremio-benchmarking-methodology/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "DualBird",
   "topic": "speed",
   "fact": "The home page body text states the headline claim as 10x-30x faster data processing with 50%-90% lower costs and no changes required. The same wording appears in the page's JSON-LD organization description and in the April 28, 2026 blog post.",
   "source_type": "vendor",
   "source_url": "https://dualbird.io/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "DualBird",
   "topic": "speed",
   "fact": "The raw HTML of the home page has a different range in its meta description and og:description (and twitter:description): 10x-100x faster. This is the meta tag, not body text. The February 5, 2026 funding blog post also says 10-100x faster and 50-90% cost savings. The body text of the home and product pages says 10x-30x, so the two ranges differ by site location.",
   "source_type": "vendor",
   "source_url": "https://dualbird.io/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "DualBird",
   "topic": "speed",
   "fact": "The product page repeats the 10x-30x end-to-end figure labeled 'Instance for instance' and 50%-90% cost reduction labeled 'Workload for workload'. The meta description of the product page is only 'Plug-and-play Spark acceleration.'",
   "source_type": "vendor",
   "source_url": "https://dualbird.io/product",
   "read_date": "2026-11-10"
  },
  {
   "platform": "DualBird",
   "topic": "stack",
   "fact": "The home page says the plugin is compatible with Apache Spark, Amazon EMR and Amazon EKS. It lists 'Optimized out of the box', 'Migration-free' and 'No more performance tuning and troubleshooting'. No Spark version numbers, Databricks support or Iceberg/Delta/Hudi support list are stated on the home, product, about or resources pages. Iceberg appears only in the compaction benchmark and in the meta description of the Dec 1, 2025 demo post ('Accelerate Apache Spark and Apache Iceberg').",
   "source_type": "vendor",
   "source_url": "https://dualbird.io/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "DualBird",
   "topic": "rewrite",
   "fact": "Onboarding is two changes: switch the EC2 instance type to F2 and add the DualBird Spark plugin. The page shows three steps: change instance type, use our Spark plugin, see the results. The about page says 'No infrastructure changes. No code changes.' The compaction blog says Spark still handles planning, scheduling and fault tolerance and no pipeline rewrite or platform migration is needed.",
   "source_type": "vendor",
   "source_url": "https://dualbird.io/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "DualBird",
   "topic": "rewrite",
   "fact": "A DualBird engineer states the FPGA path accelerates a defined set of operators (sort, join, aggregation and others). Workloads dominated by Python UDFs or unsupported operators fall back to CPU and do not see the same benefit yet. Coverage is described as expanding.",
   "source_type": "vendor",
   "source_url": "https://dualbird.io/resources/blog/a-simpler-spark",
   "read_date": "2026-11-10"
  },
  {
   "platform": "DualBird",
   "topic": "tune",
   "fact": "DualBird claims tens of Spark settings (executor sizing, CPU-memory ratios, parallelism) become irrelevant, disk spills effectively disappear and data skew stops causing spills, because only a few tasks run at once on the FPGA. The product page says the majority of common Spark performance configurations do not exist when using the solution. Cluster sizing reduces to the number of F2 nodes. These are vendor claims, not independently tested. Part 2 of the series (Sep 1, 2026) says skew still costs time, but less, and that it does not need repartitioning or salting.",
   "source_type": "vendor",
   "source_url": "https://dualbird.io/product",
   "read_date": "2026-11-10"
  },
  {
   "platform": "DualBird",
   "topic": "profile",
   "fact": "Use cases on the home page: AI data infrastructure (feature engineering and AI data prep), big data pipelines, and accelerating data analytics (fresher data for dashboards). The Iceberg compaction benchmark is the one worked example with published method.",
   "source_type": "vendor",
   "source_url": "https://dualbird.io/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "DualBird",
   "topic": "speed",
   "fact": "Home and product pages show a '110GB Enrichment Pipeline (JOIN Denormalization)' benchmark: 50%-85% lower EC2 cost with 6x-18x higher performance ceiling. The pages give no hardware, baseline, dates or method, and no link to a write-up was found.",
   "source_type": "vendor",
   "source_url": "https://dualbird.io/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "DualBird",
   "topic": "speed",
   "fact": "Sort-based Iceberg compaction on 100 GB, compared with vanilla Spark and with an unnamed 'C++ accelerated Spark' (unnamed) with tuned EBS, memory and parallelism at each cluster size. Dataset is synthetic: about 370 million rows, 64 mixed-type columns, ZSTD level 3. Results: roughly 12-20x faster single-task execution, 55-85% lower EC2 cost, stable cost across scaling, about 6x higher practical performance ceiling. Cost per terabyte compacted is stated as about $2/TB for DualBird versus about $6/TB rising past $13/TB for CPU clusters. Part of the saving is attributed to F2 local NVMe replacing EBS volumes. The home page shows 'X15-X20' single-task speedup for the same benchmark, while the June 11, 2026 blog and the white paper landing page say 12-20x. The two do not match.",
   "source_type": "vendor",
   "source_url": "https://dualbird.io/resources/blog/iceberg-compaction-benchmark-faster-spark-lower-cost",
   "read_date": "2026-11-10"
  },
  {
   "platform": "DualBird",
   "topic": "open",
   "fact": "The white paper (dated July 1, 2026) has a landing page that restates the results and says methodology, system architecture and detailed analysis are in a download. The full document is gated behind a form and was not readable. The blog gives dataset shape and baselines but not instance counts, Spark version, configs or scripts. No public repo was found (github.com/dualbird returns 404).",
   "source_type": "vendor",
   "source_url": "https://dualbird.io/resources/blog/spark-iceberg-compaction-benchmark-whitepaper",
   "read_date": "2026-11-10"
  },
  {
   "platform": "DualBird",
   "topic": "speed",
   "fact": "The 'Single Task Performance' home-page tile says X15-X20 speedup for the same cluster size while reducing cost by over 50%. The blog describes the single-task comparison as against vanilla Spark, and the April 28 post as against 'a high-end C++ columnar engine running on a single CPU core'. The baseline wording is therefore not consistent across pages.",
   "source_type": "vendor",
   "source_url": "https://dualbird.io/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "DualBird",
   "topic": "where",
   "fact": "DualBird needs AWS EC2 F2 instances. AWS describes F2 as second-generation FPGA instances with up to 8 AMD Virtex UltraScale+ HBM VU47P FPGAs, 16 GiB HBM per FPGA, 3rd-gen AMD EPYC (Milan) CPUs up to 192 vCPU, up to 2 TiB memory and up to 7.6 TiB NVMe. Sizes: f2.6xlarge (1 FPGA, 24 vCPU, 256 GiB), f2.12xlarge (2 FPGAs, 48 vCPU, 512 GiB), f2.48xlarge (8 FPGAs, 192 vCPU, 2,048 GiB).",
   "source_type": "cloud-provider-docs",
   "source_url": "https://aws.amazon.com/ec2/instance-types/f2/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "DualBird",
   "topic": "where",
   "fact": "The AWS 'instance types by Region' doc lists F2 in eight regions: us-east-1 (N. Virginia), us-west-2 (Oregon), ap-northeast-1 (Tokyo), ap-northeast-2 (Seoul), ap-southeast-2 (Sydney), ca-central-1 (Canada Central), eu-central-1 (Frankfurt) and eu-west-2 (London). F2 is not listed in other regions on that page (for example us-east-2, eu-west-1). DualBird is AWS-only: no other cloud is mentioned.",
   "source_type": "cloud-provider-docs",
   "source_url": "https://docs.aws.amazon.com/ec2/latest/instancetypes/ec2-instance-regions.html",
   "read_date": "2026-11-10"
  },
  {
   "platform": "DualBird",
   "topic": "where",
   "fact": "The product page says deployment runs on the customer's own servers in their AWS environment and 'Everything stays under your control'.",
   "source_type": "vendor",
   "source_url": "https://dualbird.io/product",
   "read_date": "2026-11-10"
  },
  {
   "platform": "DualBird",
   "topic": "breadth",
   "fact": "DualBird's own blog (Feb 5, 2026) announces a $25M raise led by Lightspeed Venture Partners with Angular Ventures, Uncork Capital and Bessemer Venture Partners. It says general availability was planned for early 2026 and that early deployments with design partners showed hours-to-minutes gains. The funding is confirmed by the company itself only. Lightspeed's page returned HTTP 403 and no press article was reachable.",
   "source_type": "vendor",
   "source_url": "https://dualbird.io/resources/blog/dualbird-raises-25m",
   "read_date": "2026-11-10"
  },
  {
   "platform": "DualBird",
   "topic": "profile",
   "fact": "About page: founded and led by semiconductor and distributed data systems veterans; positions FPGA-based acceleration as a drop-in Spark plugin on standard AWS instances. Blog posts date from Dec 1, 2025 to Sep 1, 2026. Footer copyright reads 2025. Contact: contact@dualbird.io.",
   "source_type": "vendor",
   "source_url": "https://dualbird.io/about",
   "read_date": "2026-11-10"
  },
  {
   "platform": "DualBird",
   "topic": "cost",
   "fact": "No pricing, price list or marketplace listing was found on the site. Cost claims are only percentages (50%-90% lower, 55%-85% lower EC2) and the about $2/TB versus $6-13/TB compaction figure. The pricing model for the plugin itself is not stated, so users cannot tell whether F2 instance price or license fee is included in the claimed savings.",
   "source_type": "vendor",
   "source_url": "https://dualbird.io/resources/blog/iceberg-compaction-benchmark-faster-spark-lower-cost",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Google BigQuery and BigLake",
   "topic": "where",
   "fact": "BigQuery Omni lets BigQuery analytics run on data in Amazon S3 through BigLake tables, in AWS regions us-east-1, us-west-2, ap-northeast-2, ap-southeast-2, eu-west-1 and eu-central-1.",
   "source_type": "cloud-provider-docs",
   "source_url": "https://cloud.google.com/bigquery/docs/omni-introduction",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Google BigQuery and BigLake",
   "topic": "rewrite",
   "fact": "On BigQuery Omni, DML statements and CREATE TABLE are not supported, BigQuery ML statements are not supported, and only EXPORT DATA queries are allowed for destination tables, so it does not run write-heavy Spark batch ETL on S3. Standard and Enterprise Plus editions do not support Omni regions.",
   "source_type": "cloud-provider-docs",
   "source_url": "https://cloud.google.com/bigquery/docs/omni-introduction",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Google BigQuery and BigLake",
   "topic": "rewrite",
   "fact": "BigQuery stored procedures for Apache Spark let you run PySpark or Scala/Java JAR code from SQL via CREATE PROCEDURE ... OPTIONS(engine = \"SPARK\", runtime_version = \"2.2\"), using a Cloud Storage file or inline code and a connection; this runs on Google Cloud, so code must be wrapped as procedures.",
   "source_type": "cloud-provider-docs",
   "source_url": "https://cloud.google.com/bigquery/docs/spark-procedures",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Google BigQuery and BigLake",
   "topic": "stack",
   "fact": "BigQuery Iceberg managed tables store data in buckets you own, support DML, streaming and Spark connectors, export Iceberg V2 snapshots for open engines, and do automatic compaction, clustering and garbage collection.",
   "source_type": "cloud-provider-docs",
   "source_url": "https://cloud.google.com/bigquery/docs/iceberg-tables",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Google BigQuery and BigLake",
   "topic": "stack",
   "fact": "BigLake tables let you query structured data in Cloud Storage, Amazon S3 and Blob Storage with access delegation and row and column security.",
   "source_type": "cloud-provider-docs",
   "source_url": "https://cloud.google.com/bigquery/docs/biglake-intro",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Google BigQuery and BigLake",
   "topic": "cost",
   "fact": "BigQuery offers on-demand pricing per TiB scanned (first 1 TiB per month free) or capacity pricing per slot-hour. BigQuery Omni has its own on-demand per TiB pricing for AWS regions. The page text shows $0.06 per slot-hour for Enterprise edition pay-as-you-go compute, with no region shown beside the rate (read 2026-11-10).",
   "source_type": "vendor-docs",
   "source_url": "https://cloud.google.com/bigquery/pricing",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Google BigQuery and BigLake",
   "topic": "cost",
   "fact": "Google's Spark service (Managed Service for Apache Spark, formerly Dataproc) bills serverless Spark per second in Data Compute Units with a 1-minute minimum, plus shuffle storage; Standard tier is for batch ETL, Premium tier uses Lightning Engine.",
   "source_type": "vendor-docs",
   "source_url": "https://cloud.google.com/dataproc-serverless/pricing",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Google BigQuery and BigLake",
   "topic": "speed",
   "fact": "Google states Lightning Engine delivers up to 4.9x faster performance than open source Spark and up to 2.5x the price-performance over a high-speed Spark offering Google does not name; no method is given on the pricing page.",
   "source_type": "vendor",
   "source_url": "https://cloud.google.com/dataproc-serverless/pricing",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Google BigQuery and BigLake",
   "topic": "tune",
   "fact": "Iceberg managed tables run automatic storage optimization (adaptive file sizing, clustering, garbage collection), so compaction needs no job configuration on Google Cloud; storage optimization is a separate billed item.",
   "source_type": "cloud-provider-docs",
   "source_url": "https://cloud.google.com/bigquery/docs/iceberg-tables",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Microsoft Fabric",
   "topic": "where",
   "fact": "Fabric workloads run over OneLake, described as built on Azure Data Lake Storage; the region availability page lists Azure regions and no AWS regions, so Spark compute does not run inside AWS.",
   "source_type": "vendor-docs",
   "source_url": "https://learn.microsoft.com/en-us/fabric/fundamentals/microsoft-fabric-overview",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Microsoft Fabric",
   "topic": "where",
   "fact": "OneLake S3 shortcuts give read-only access to data in an S3 bucket in any file format without copying; file caching can reduce cross-cloud egress costs. Writes to S3 are not possible through the shortcut.",
   "source_type": "vendor-docs",
   "source_url": "https://learn.microsoft.com/en-us/fabric/onelake/create-s3-shortcut",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Microsoft Fabric",
   "topic": "cost",
   "fact": "The Azure retail price API lists Fabric Spark capacity usage at $0.18 per CU-hour for eastus (Spark Memory Optimized and autoscale for Spark entries, pay-as-you-go).",
   "source_type": "vendor-docs",
   "source_url": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Microsoft%20Fabric'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Microsoft Fabric",
   "topic": "cost",
   "fact": "In Fabric each capacity unit maps to two Spark VCores, and Spark jobs are admitted by available VCores with FIFO queueing and throttling.",
   "source_type": "vendor-docs",
   "source_url": "https://learn.microsoft.com/en-us/fabric/data-engineering/spark-job-concurrency-and-queueing",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Microsoft Fabric",
   "topic": "cost",
   "fact": "Autoscale Billing for Spark is an opt-in pay-as-you-go model that runs Spark on dedicated serverless resources at standard PAYG CU-hour rates; reserved capacity discounts do not apply and a base capacity is still required.",
   "source_type": "vendor-docs",
   "source_url": "https://azure.microsoft.com/en-us/pricing/details/microsoft-fabric/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Microsoft Fabric",
   "topic": "rewrite",
   "fact": "The Fabric native execution engine, based on Apache Gluten and Velox, needs no code changes, falls back to JVM Spark for unsupported operators, and supports Parquet and Delta.",
   "source_type": "vendor-docs",
   "source_url": "https://learn.microsoft.com/en-us/fabric/data-engineering/runtime",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Microsoft Fabric",
   "topic": "speed",
   "fact": "Microsoft reports that on TPC-DS scale factor 1000 with Delta the native engine was up to six times faster than open-source Spark, with about 83% compute cost savings on a fixed-size cluster.",
   "source_type": "vendor",
   "source_url": "https://learn.microsoft.com/en-us/fabric/data-engineering/runtime",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Microsoft Fabric",
   "topic": "stack",
   "fact": "Fabric Runtime 1.3 uses Apache Spark 3.5 and Runtime 2.0 uses Spark 4.1; OneLake handles Delta and Iceberg through metadata virtualization so Iceberg tables read as Delta and vice versa.",
   "source_type": "vendor-docs",
   "source_url": "https://learn.microsoft.com/en-us/fabric/onelake/onelake-iceberg-tables",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Microsoft Fabric",
   "topic": "tune",
   "fact": "Starter pools start in 5 to 10 seconds with no setup; custom Spark pools let you tune node size and scaling, and native execution can be toggled per application with a Spark setting.",
   "source_type": "vendor-docs",
   "source_url": "https://learn.microsoft.com/en-us/fabric/data-engineering/spark-compute",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Onehouse (Onehouse Compute Runtime with the Quanton engine)",
   "topic": "rewrite",
   "fact": "Onehouse docs say its Jobs are fully compatible with Apache Spark and that EMR code can be reused unless it uses AWS-specific features. Migration work listed: map EMR steps to Onehouse Jobs created through its API or Airflow operators (CREATE CLUSTER, CREATE JOB, RUN JOB, DELETE CLUSTER), and configure Glue catalog sync or contact support to make Glue the primary catalog.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.onehouse.ai/product/jobs/emr-migration",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Onehouse (Onehouse Compute Runtime with the Quanton engine)",
   "topic": "rewrite",
   "fact": "At the May 20, 2025 launch Onehouse said users can run existing Spark and SQL jobs on new cluster types and that existing Hudi/Spark jobs can be pointed at Onehouse 'with no code changes'. The Quanton Kubernetes Operator is described as a 'drop-in replacement' for an existing Spark Kubernetes operator.",
   "source_type": "vendor",
   "source_url": "https://www.onehouse.ai/blog/announcing-spark-and-sql-on-the-onehouse-compute-runtime-with-quanton",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Onehouse (Onehouse Compute Runtime with the Quanton engine)",
   "topic": "cost",
   "fact": "Onehouse bills in Onehouse Compute Units (OCU) on a bring-your-own-cloud model: 1 OCU is a 4-core instance for 1 hour in your account, plus support costs; no separate volume or storage fee, and the customer pays its own AWS or GCP infrastructure bill. The docs page does not state a dollar rate per OCU.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.onehouse.ai/product/project-settings/usage-and-billing",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Onehouse (Onehouse Compute Runtime with the Quanton engine)",
   "topic": "cost",
   "fact": "Vendor claims at the Quanton launch: 2-3x better price-performance for ETL, and a guaranteed greater-than-50% cost reduction for existing Hudi users moving Spark jobs from AWS EMR or open-source Spark. These are guarantees stated by the vendor, not results from an independent party.",
   "source_type": "vendor",
   "source_url": "https://www.onehouse.ai/blog/announcing-spark-and-sql-on-the-onehouse-compute-runtime-with-quanton",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Onehouse (Onehouse Compute Runtime with the Quanton engine)",
   "topic": "where",
   "fact": "Onehouse runs EC2 or Google Compute Engine instances inside the customer's AWS or GCP account (Azure not listed in the billing docs). Standard instance types map to Graviton m8g sizes on AWS (oh-general-4 = m8g.xlarge, oh-general-8 = m8g.2xlarge, oh-general-16 = m8g.4xlarge). A project is scoped to one region and account; adding a region needs a support ticket.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.onehouse.ai/product/project-settings/usage-and-billing",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Onehouse (Onehouse Compute Runtime with the Quanton engine)",
   "topic": "stack",
   "fact": "Quanton runs Spark and SQL jobs that read and write Hudi, Iceberg and Delta Lake. Docs: Iceberg 1.10.0 is installed automatically when a job uses an Iceberg REST Catalog (AWS Glue and Snowflake Open Catalog are documented); for Delta Lake the user installs Delta Spark as a dependency in the JAR or Python job. In the Nov 12, 2025 Iceberg post Onehouse said Delta acceleration is 'up next'.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.onehouse.ai/product/jobs/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Onehouse (Onehouse Compute Runtime with the Quanton engine)",
   "topic": "stack",
   "fact": "Apache XTable (formerly OneTable) translates table metadata between Hudi, Delta Lake and Iceberg without being a new format. It is incubating at the Apache Software Foundation; Onehouse says it created Hudi and XTable.",
   "source_type": "open-source-project",
   "source_url": "https://xtable.apache.org/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Onehouse (Onehouse Compute Runtime with the Quanton engine)",
   "topic": "tune",
   "fact": "Each cluster needs min and max OCU limits. Docs warn that if usage reaches Max OCU the cluster stops scaling up, which can delay processing; Min OCU keeps a cluster warm. The Onehouse docs pages read give no per-job Spark tuning list; Onehouse claims automatic scaling and automated async table services.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.onehouse.ai/product/compute/create-cluster",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Onehouse (Onehouse Compute Runtime with the Quanton engine)",
   "topic": "speed",
   "fact": "Onehouse benchmarked Quanton against open-source Spark on Iceberg: TPC-DS up to about 3.1x, TPCx-BB 2x, TPC-DI (1 TB) 2x with 3-4x on MERGE INTO, and LakeLoader 4.1x for writes. The baseline is open-source Spark, not EMR.",
   "source_type": "vendor",
   "source_url": "https://www.onehouse.ai/blog/apache-iceberg-on-quanton-3x-faster-apache-spark-workloads",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Onehouse (Onehouse Compute Runtime with the Quanton engine)",
   "topic": "speed",
   "fact": "Onehouse reran its benchmarks against EMR 7.12 after AWS announced it: it reports EMR improved 32% over earlier versions and that Onehouse 'still comes out with a 2.5x better price/performance advantage'. Setup: 10 TB TPC-DS for read/transform, 1 TB LakeLoader merges for load, latest general-purpose Graviton instances. Results are shown as charts; Onehouse also reports Hudi and Delta query performance on EMR 7.12 was very similar to Iceberg.",
   "source_type": "vendor",
   "source_url": "https://www.onehouse.ai/blog/onehouse-quanton-vs-the-latest-aws-emr-for-apache-spark-workloads",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Onehouse (Onehouse Compute Runtime with the Quanton engine)",
   "topic": "speed",
   "fact": "A B2B SaaS customer (unnamed) ran its UDF-heavy pipeline on EMR and on Quanton in its own AWS account on m8gd nodes. Quanton was faster at p50, p97 and p99 tenant sizes; on the largest tenant (21 billion rows) EMR filled local disks on 180 nodes and was aborted after 3.7 hours, and finished on 300 nodes. The write-up is published by Onehouse.",
   "source_type": "vendor",
   "source_url": "https://www.onehouse.ai/blog/emr-migration-to-quanton-spark-udf-jobs",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Onehouse (Onehouse Compute Runtime with the Quanton engine)",
   "topic": "open",
   "fact": "Onehouse open-sourced LakeLoader, a tool that generates incremental update/insert loads for benchmarking write performance. The repository is Apache-2.0 licensed, last pushed 2026-10-30. The Quanton engine itself and the full Iceberg benchmark configurations were not found as a repository in the pages read.",
   "source_type": "open-source-project",
   "source_url": "https://api.github.com/repos/onehouseinc/lake-loader",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Onehouse (Onehouse Compute Runtime with the Quanton engine)",
   "topic": "profile",
   "fact": "Onehouse was founded by Vinoth Chandar, creator and PMC chair of Apache Hudi, who built the first data lakehouse at Uber. Its product claims also include a Table Optimizer, OneSync catalog sync, and a Quanton Kubernetes Operator to run Quanton Spark on the customer's own Kubernetes.",
   "source_type": "vendor",
   "source_url": "https://www.onehouse.ai/about-us",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Open stack: Spark on EMR or EKS with Iceberg, Delta or Hudi",
   "topic": "cost",
   "fact": "EMR on EC2 charges an EMR price in addition to the EC2 price (and EBS), billed per second with a one-minute minimum. AWS's example in us-east-1: c4.2xlarge EMR charge $0.105/hour next to EC2 $0.398/hour; one master and two core nodes for a month total $1,101.57. Spot instances are up to a 90% discount versus On-Demand.",
   "source_type": "cloud-provider-docs",
   "source_url": "https://aws.amazon.com/emr/pricing/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Open stack: Spark on EMR or EKS with Iceberg, Delta or Hudi",
   "topic": "cost",
   "fact": "EMR on EKS adds an uplift per vCPU-hour and per GB-hour to EKS/EC2 or Fargate costs, plus $0.10 per hour for each EKS cluster. AWS's worked example uses $0.007084 per vCPU-hour and $0.000777875 per GB-hour (100 vCPU and 300 GB for 30 minutes = $0.4709 EMR uplift). The uplift applies to any pod using an EMR runtime image, including with the Spark operator.",
   "source_type": "cloud-provider-docs",
   "source_url": "https://aws.amazon.com/emr/pricing/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Open stack: Spark on EMR or EKS with Iceberg, Delta or Hudi",
   "topic": "cost",
   "fact": "EMR Serverless is billed per vCPU-hour and GB-hour. AWS's worked example uses $0.052624 per vCPU-hour and $0.0057785 per GB-hour for a job with 100 vCPU and 750 GB for 30 minutes, then 200 more vCPU for 15 minutes, totaling $9.596275.",
   "source_type": "cloud-provider-docs",
   "source_url": "https://aws.amazon.com/emr/pricing/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Open stack: Spark on EMR or EKS with Iceberg, Delta or Hudi",
   "topic": "rewrite",
   "fact": "The EMR runtime for Spark provides 'full API compatibility with open source Apache Spark and Apache Iceberg', so existing Spark jobs run as is. Optimized runtimes are used by EMR on EC2, EMR Serverless, EMR on EKS, EMR on Outposts and AWS Glue. Self-run Spark on EKS uses the open-source Spark operator and needs no job rewrite.",
   "source_type": "vendor",
   "source_url": "https://aws.amazon.com/blogs/big-data/run-apache-spark-and-iceberg-4-5x-faster-than-open-source-spark-with-amazon-emr/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Open stack: Spark on EMR or EKS with Iceberg, Delta or Hudi",
   "topic": "stack",
   "fact": "Amazon EMR 7.14.0 packages Spark 3.5.8-amzn-0, Delta 3.3.2-amzn-3, Iceberg 1.10.1-amzn-1 and Hudi 1.0.2-amzn-3. All three table formats are available on EMR.",
   "source_type": "cloud-provider-docs",
   "source_url": "https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-release-app-versions-7.x.html",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Open stack: Spark on EMR or EKS with Iceberg, Delta or Hudi",
   "topic": "stack",
   "fact": "EMR on EKS supports the Kubernetes Operator for Apache Spark (Spark operator) as a job submission model with the EMR runtime on your own EKS clusters. The Kubeflow Spark Operator is Apache-2.0 licensed with Helm installation.",
   "source_type": "cloud-provider-docs",
   "source_url": "https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/spark-operator.html",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Open stack: Spark on EMR or EKS with Iceberg, Delta or Hudi",
   "topic": "tune",
   "fact": "Iceberg docs say users must run maintenance: expire snapshots, remove old metadata files and delete orphan files are 'recommended'; compacting data files with the Spark rewriteDataFiles action (for example target-file-size-bytes of 500 MB) and rewriting manifests are 'optional' maintenance, run as jobs you schedule.",
   "source_type": "open-source-project",
   "source_url": "https://iceberg.apache.org/docs/latest/maintenance/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Open stack: Spark on EMR or EKS with Iceberg, Delta or Hudi",
   "topic": "tune",
   "fact": "Hudi merge-on-read tables need compaction to merge log files into base files and bound read latency. Compaction has two steps, scheduling (plan written to the timeline) and execution; the docs describe strategies for choosing which file slices to compact.",
   "source_type": "open-source-project",
   "source_url": "https://hudi.apache.org/docs/compaction",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Open stack: Spark on EMR or EKS with Iceberg, Delta or Hudi",
   "topic": "tune",
   "fact": "Delta Lake OSS docs list file management features the user applies: compaction (bin-packing) with OPTIMIZE, auto compaction, data skipping, Z-ordering, multi-part checkpointing, log compactions and optimized write.",
   "source_type": "open-source-project",
   "source_url": "https://docs.delta.io/latest/optimizations-oss.html",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Open stack: Spark on EMR or EKS with Iceberg, Delta or Hudi",
   "topic": "where",
   "fact": "Amazon EMR has regional endpoints in the US (us-east-1, us-east-2, us-west-1, us-west-2) and many other AWS regions per the AWS general reference endpoints table. Spark runs on EC2 instances the user chooses (the AWS benchmark used r5d.4xlarge).",
   "source_type": "cloud-provider-docs",
   "source_url": "https://docs.aws.amazon.com/general/latest/gr/emr.html",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Open stack: Spark on EMR or EKS with Iceberg, Delta or Hudi",
   "topic": "speed",
   "fact": "AWS ran TPC-DS 3 TB (104 SparkSQL queries, 3 rounds, 9 x r5d.4xlarge, Hadoop catalog) and reports EMR 7.12 averaged 1,349.62 s versus 6,113.92 s for open-source Spark 3.5.6 with Iceberg 1.10.0 and 1,535.62 s for EMR 7.5. AWS notes results are not comparable to official TPC-DS. The run is by the vendor of EMR.",
   "source_type": "vendor",
   "source_url": "https://aws.amazon.com/blogs/big-data/run-apache-spark-and-iceberg-4-5x-faster-than-open-source-spark-with-amazon-emr/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Open stack: Spark on EMR or EKS with Iceberg, Delta or Hudi",
   "topic": "cost",
   "fact": "In the same AWS benchmark the cost was $4.81 for EMR 7.12, $5.47 for EMR 7.5 and $17.65 for open-source Spark 3.5.6 with Iceberg 1.10.0. Cost formula uses r5d.4xlarge at $1.152/hour plus $0.27/hour EMR plus EBS, On-Demand.",
   "source_type": "vendor",
   "source_url": "https://aws.amazon.com/blogs/big-data/run-apache-spark-and-iceberg-4-5x-faster-than-open-source-spark-with-amazon-emr/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Open stack: Spark on EMR or EKS with Iceberg, Delta or Hudi",
   "topic": "open",
   "fact": "AWS publishes the setup in GitHub repositories (aws-samples/emr-on-eks-benchmark branch tpcds-v2.13_iceberg, and aws-samples/emr-spark-benchmark) with build instructions, so the TPC-DS Iceberg benchmark can be repeated.",
   "source_type": "vendor",
   "source_url": "https://github.com/aws-samples/emr-on-eks-benchmark",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Snowflake",
   "topic": "rewrite",
   "fact": "Snowpark Connect for Apache Spark runs existing PySpark DataFrame code against the Snowflake engine over the Spark Connect protocol; there is no Spark cluster, all workloads run on a Snowflake warehouse. Batch jobs are submitted with the Snowpark Submit CLI.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.snowflake.com/en/developer-guide/snowpark-connect/snowpark-connect-apache-spark",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Snowflake",
   "topic": "stack",
   "fact": "Snowpark Connect supports Apache Spark 3.5 only; workloads written for 3.4 and below or 4.0 and above are not supported and may produce protocol errors or missing API failures.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.snowflake.com/en/developer-guide/snowpark-connect/snowpark-connect-limitations.md",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Snowflake",
   "topic": "rewrite",
   "fact": "Because Snowflake rather than Spark is the engine, there are semantic differences: DayTimeIntervalType, YearMonthIntervalType and user-defined types are unsupported, and ByteType, ShortType and IntegerType are returned as LongType. The docs list unsupported APIs for DataFrame and Dataset.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.snowflake.com/en/developer-guide/snowpark-connect/snowpark-connect-compatibility.md",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Snowflake",
   "topic": "rewrite",
   "fact": "Docs say the Spark DataFrame and Spark SQL code can be reused with minimal migration or changes; the compatibility notes say Spark Connect defers plan resolution, which changes behavior versus Spark Classic.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.snowflake.com/en/developer-guide/snowpark-connect/snowpark-connect-apache-spark",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Snowflake",
   "topic": "cost",
   "fact": "Warehouses bill credits per second with a 60-second minimum on each start; each size step roughly doubles credits per hour (Small is 2 credits per hour, Medium is 4). Snowflake's Service Consumption Table (effective 2026-11-09) lists on-demand credits on AWS US East at $2.00 (Standard), $3.00 (Enterprise) and $4.00 (Business Critical).",
   "source_type": "vendor-docs",
   "source_url": "https://docs.snowflake.com/en/user-guide/cost-understanding-compute",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Snowflake",
   "topic": "stack",
   "fact": "Snowflake supports Iceberg tables (Parquet) with data in customer-managed S3, GCS or Azure storage via external volumes, an external catalog integration such as AWS Glue, and Delta files read through Delta Direct; Hudi was not listed on the page.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.snowflake.com/en/user-guide/tables-iceberg",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Snowflake",
   "topic": "where",
   "fact": "Snowflake runs on AWS, Azure and Google Cloud; AWS regions include us-east-1, us-west-2, eu-west-1 and ca-central-1. Each account is hosted in a single region.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.snowflake.com/en/user-guide/intro-regions.md",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Snowflake",
   "topic": "tune",
   "fact": "Docs provide a separate Optimizing Snowpark Connect for Spark workloads section and a Troubleshooting section covering known limitations, so some job-level adjustment is expected after migration.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.snowflake.com/en/developer-guide/snowpark-connect/snowpark-connect-troubleshooting.md",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Starburst (Starburst Galaxy, Starburst Enterprise)",
   "topic": "cost",
   "fact": "Galaxy credit prices start at $0.50 (Pro), $0.75 (Enterprise) and $1.00 (Mission Critical) per credit. A Free tier allows up to 3 clusters. Pricing depends on compute usage, tier and the cloud provider and region. Annual commitments may get discounts.",
   "source_type": "vendor",
   "source_url": "https://www.starburst.io/pricing/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Starburst (Starburst Galaxy, Starburst Enterprise)",
   "topic": "cost",
   "fact": "On-demand list prices per credit (Pro / Enterprise / Mission Critical) on AWS: US East N. Virginia, US East Ohio and US West Oregon $0.50 / $0.75 / $1.00; Europe Ireland $0.56 / $0.84 / $1.12; Europe Frankfurt $0.60 / $0.90 / $1.20; Asia Pacific Tokyo $0.64 / $0.97 / $1.29; Brazil (Sao Paulo) $0.80 / $1.19 / $1.59.",
   "source_type": "vendor",
   "source_url": "https://www.starburst.io/pricing/pricing-details/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Starburst (Starburst Galaxy, Starburst Enterprise)",
   "topic": "cost",
   "fact": "Cluster compute credit consumption: 1 worker 6 credits per hour, 2 workers 12, 4 workers 24, 8 workers 48, 16 workers 96. Managed ingest: file ingest 0.60 credits per hour; table maintenance and data transforms 0.30 credits per hour. Billing tracks cluster compute hours, data ingestion compute and private link data transfer.",
   "source_type": "vendor",
   "source_url": "https://www.starburst.io/pricing/pricing-details/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Starburst (Starburst Galaxy, Starburst Enterprise)",
   "topic": "where",
   "fact": "Galaxy prices are listed for AWS, Azure and GCP regions. AWS regions priced include us-east-1, us-east-2, us-west-1, us-west-2, ca-central, Ireland, London, Frankfurt, Zurich, Singapore, Sydney, Tokyo, Cape Town, UAE, Tel Aviv and Sao Paulo. A cluster and its catalogs are normally in the same cloud and region; cross-region querying is a private preview feature.",
   "source_type": "vendor",
   "source_url": "https://www.starburst.io/pricing/pricing-details/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Starburst (Starburst Galaxy, Starburst Enterprise)",
   "topic": "rewrite",
   "fact": "Galaxy is 'a fully-managed data lake analytics platform built in addition to Trino'. Work runs as SQL on Trino, not Spark, so Spark jobs (DataFrame code, UDFs, PySpark) would have to be rewritten as Trino SQL, or kept on Spark against the same Iceberg tables. The Galaxy docs read contain no Spark execution mode.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.starburst.io/starburst-galaxy/index.html",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Starburst (Starburst Galaxy, Starburst Enterprise)",
   "topic": "stack",
   "fact": "Galaxy's Great Lakes connectivity lists Iceberg, Delta Lake, Hive and Hudi table format pages. Hudi tables are read-only: 'Galaxy cannot create new Hudi tables or write to them.' Iceberg tables can be created and written; the Iceberg data file format can be Parquet, ORC or Avro.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.starburst.io/starburst-galaxy/data-engineering/working-with-data-lakes/table-formats/gl-hudi.html",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Starburst (Starburst Galaxy, Starburst Enterprise)",
   "topic": "tune",
   "fact": "Iceberg table maintenance in Galaxy is run with ALTER TABLE ... EXECUTE commands: optimize (compaction; files under file_size_threshold, default 100MB, are merged), drop_extended_stats, expire_snapshots, remove_orphan_files. Live table maintenance for ingest tables is a separate Galaxy page.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.starburst.io/starburst-galaxy/data-engineering/working-with-data-lakes/table-formats/gl-iceberg.html",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Starburst (Starburst Galaxy, Starburst Enterprise)",
   "topic": "tune",
   "fact": "For batch-style work Galaxy offers a fault tolerant execution mode that spools intermediate data and retries failed parts of a query. Docs say 'No other configuration is required' to enable it, but not to use it if most queries run under about a minute, and that it can be slightly slower than normal execution. FTE and Accelerated modes are not available on the Free cluster size and not all catalogs support FTE.",
   "source_type": "vendor-docs",
   "source_url": "https://docs.starburst.io/starburst-galaxy/data-engineering/optimization-performance-and-quality/workload-optimization/clusters-with-fte.html",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Starburst (Starburst Galaxy, Starburst Enterprise)",
   "topic": "speed",
   "fact": "Starburst reported GPU-accelerated Trino results (its proprietary distribution, not open-source Trino): TPC-H up to 6x on accelerated queries and ClickBench 3.4x overall, single-node development environment. This is a SQL query test, not Spark batch work.",
   "source_type": "vendor",
   "source_url": "https://www.starburst.io/blog/gpu-accelerated-sql-analytics-how-starburst-and-nvidia-deliver-industry-benchmark-speedups-on-gpu-infrastructure/",
   "read_date": "2026-11-10"
  },
  {
   "platform": "Starburst (Starburst Galaxy, Starburst Enterprise)",
   "topic": "profile",
   "fact": "Starburst's August 7, 2026 blog on SAP's acquisition of Dremio is competitor commentary about SAP's acquisition of Dremio; it does not announce any change to Starburst's own ownership.",
   "source_type": "vendor",
   "source_url": "https://www.starburst.io/blog/5-questions-every-dremio-customer-should-ask-about-the-sap-deal/",
   "read_date": "2026-11-10"
  }
 ]
}