[{"data":1,"prerenderedAt":73},["ShallowReactive",2],{"doc:\u002Fdatabases\u002Fformats":3},{"page":4,"toc":69},{"path":5,"title":6,"description":7,"blocks":8},"\u002Fdatabases\u002Fformats","MMDB, CSV & Parquet","How the three database file formats are structured and which one to pick for your workload.",[9,12,35,40,42,45,47,55,58],{"type":10,"text":11},"p","Every database subscription unlocks three synchronized artifacts for each snapshot: `.mmdb` for real-time application lookups, `.csv.gz` for relational bulk loading, and `.parquet` for analytical warehouses.",{"type":13,"columns":14,"rows":19},"table",[15,16,17,18],"Format","Extension","Best for","Typical lookup \u002F query speed",[20,25,30],[21,22,23,24],"**MMDB**","`.mmdb`","Application servers, edge workers, NGINX \u002F Envoy","`\u003C 1 µs` via `mmap`",[26,27,28,29],"**Parquet**","`.parquet`","ClickHouse, DuckDB, Snowflake, BigQuery, Spark","Vectorized columnar scan",[31,32,33,34],"**CSV**","`.csv.gz`","PostgreSQL `COPY`, MySQL, custom ETL pipelines","Sequential stream",{"type":36,"level":37,"text":38,"id":39},"heading",2,"MMDB (MaxMind DB binary format v2.0)","mmdb-maxmind-db-binary-format-v2-0",{"type":10,"text":41},"A binary radix tree mapping IPv4 and IPv6 prefixes to nested dictionaries that match the [REST API response format](\u002Fgetting-started\u002Fresponse-format) (`geo`, `network`, `connection`, `security`, `security_extensions`). Opens directly in `libmaxminddb` and every standard MaxMind reader without code changes.",{"type":36,"level":37,"text":43,"id":44},"Columnar Parquet & flattened CSV columns","columnar-parquet-and-flattened-csv-columns",{"type":10,"text":46},"In `.csv.gz` and `.parquet`, nested objects are flattened with snake_case column names and `cidr` as the first column:",{"type":48,"samples":49},"code",[50],{"label":51,"lang":52,"code":53,"html":54},"CSV header (Enterprise Mega)","text","cidr,ip_version,country_code,declared_country_code,physical_country_code,is_virtual_location,location_type,geo_confidence,subdivision,city,postal_code,latitude,longitude,accuracy_radius_km,timezone,asn,organization,connection_type,is_hosting,is_residential,is_mobile,is_business,is_vpn,is_tor_exit,is_proxy,anonymizer_type,security_confidence,anonymizer_reason,threat_level,abuse_velocity\n198.51.100.0\u002F24,4,US,US,US,false,physical,0.94,Virginia,Ashburn,20147,39.0438,-77.4874,10,America\u002FNew_York,64501,\"Example Cloud LLC\",hosting,true,false,false,false,true,false,false,vpn,0.96,vpn_handshake_confirmed,medium,low","\u003Cpre class=\"shiki\">\u003Ccode>cidr,ip_version,country_code,declared_country_code,physical_country_code,is_virtual_location,location_type,geo_confidence,subdivision,city,postal_code,latitude,longitude,accuracy_radius_km,timezone,asn,organization,connection_type,is_hosting,is_residential,is_mobile,is_business,is_vpn,is_tor_exit,is_proxy,anonymizer_type,security_confidence,anonymizer_reason,threat_level,abuse_velocity\n198.51.100.0\u002F24,4,US,US,US,false,physical,0.94,Virginia,Ashburn,20147,39.0438,-77.4874,10,America\u002FNew_York,64501,\"Example Cloud LLC\",hosting,true,false,false,false,true,false,false,vpn,0.96,vpn_handshake_confirmed,medium,low\u003C\u002Fcode>\u003C\u002Fpre>",{"type":36,"level":37,"text":56,"id":57},"Querying Parquet directly with DuckDB or ClickHouse","querying-parquet-directly-with-duckdb-or-clickhouse",{"type":48,"samples":59},[60,65],{"label":61,"lang":62,"code":63,"html":64},"DuckDB","sql","SELECT country_code, count(*) AS prefixes\nFROM read_parquet('ipscope-complete.parquet')\nWHERE is_vpn = true\nGROUP BY country_code\nORDER BY prefixes DESC\nLIMIT 10;","\u003Cpre class=\"shiki ipscope\" style=\"background-color:var(--shiki-background);color:var(--shiki-foreground)\" tabindex=\"0\">\u003Ccode>\u003Cspan class=\"line\">\u003Cspan style=\"color:var(--shiki-token-keyword)\">SELECT\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-foreground)\"> country_code, \u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-token-function)\">count\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-foreground)\">(\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-token-keyword)\">*\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-foreground)\">) \u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-token-keyword)\">AS\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-foreground)\"> prefixes\u003C\u002Fspan>\u003C\u002Fspan>\n\u003Cspan class=\"line\">\u003Cspan style=\"color:var(--shiki-token-keyword)\">FROM\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-foreground)\"> read_parquet(\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-token-string-expression)\">'ipscope-complete.parquet'\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-foreground)\">)\u003C\u002Fspan>\u003C\u002Fspan>\n\u003Cspan class=\"line\">\u003Cspan style=\"color:var(--shiki-token-keyword)\">WHERE\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-foreground)\"> is_vpn \u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-token-keyword)\">=\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-foreground)\"> true\u003C\u002Fspan>\u003C\u002Fspan>\n\u003Cspan class=\"line\">\u003Cspan style=\"color:var(--shiki-token-keyword)\">GROUP BY\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-foreground)\"> country_code\u003C\u002Fspan>\u003C\u002Fspan>\n\u003Cspan class=\"line\">\u003Cspan style=\"color:var(--shiki-token-keyword)\">ORDER BY\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-foreground)\"> prefixes \u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-token-keyword)\">DESC\u003C\u002Fspan>\u003C\u002Fspan>\n\u003Cspan class=\"line\">\u003Cspan style=\"color:var(--shiki-token-keyword)\">LIMIT\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-token-constant)\"> 10\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-foreground)\">;\u003C\u002Fspan>\u003C\u002Fspan>\u003C\u002Fcode>\u003C\u002Fpre>",{"label":66,"lang":62,"code":67,"html":68},"PostgreSQL","CREATE TABLE ipscope_blocks (\n  network cidr PRIMARY KEY,\n  country_code text,\n  asn integer,\n  organization text,\n  is_vpn boolean\n);\nCREATE INDEX ON ipscope_blocks USING gist (network inet_ops);\n\nSELECT country_code, asn, is_vpn\nFROM ipscope_blocks\nWHERE network >>= inet '198.51.100.7'\nORDER BY masklen(network) DESC\nLIMIT 1;","\u003Cpre class=\"shiki ipscope\" style=\"background-color:var(--shiki-background);color:var(--shiki-foreground)\" tabindex=\"0\">\u003Ccode>\u003Cspan class=\"line\">\u003Cspan style=\"color:var(--shiki-token-keyword)\">CREATE\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-token-keyword)\"> TABLE\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-token-function)\"> ipscope_blocks\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-foreground)\"> (\u003C\u002Fspan>\u003C\u002Fspan>\n\u003Cspan class=\"line\">\u003Cspan style=\"color:var(--shiki-foreground)\">  network \u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-token-keyword)\">cidr\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-token-keyword)\"> PRIMARY KEY\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-foreground)\">,\u003C\u002Fspan>\u003C\u002Fspan>\n\u003Cspan class=\"line\">\u003Cspan style=\"color:var(--shiki-foreground)\">  country_code \u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-token-keyword)\">text\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-foreground)\">,\u003C\u002Fspan>\u003C\u002Fspan>\n\u003Cspan class=\"line\">\u003Cspan style=\"color:var(--shiki-foreground)\">  asn \u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-token-keyword)\">integer\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-foreground)\">,\u003C\u002Fspan>\u003C\u002Fspan>\n\u003Cspan class=\"line\">\u003Cspan style=\"color:var(--shiki-foreground)\">  organization \u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-token-keyword)\">text\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-foreground)\">,\u003C\u002Fspan>\u003C\u002Fspan>\n\u003Cspan class=\"line\">\u003Cspan style=\"color:var(--shiki-foreground)\">  is_vpn \u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-token-keyword)\">boolean\u003C\u002Fspan>\u003C\u002Fspan>\n\u003Cspan class=\"line\">\u003Cspan style=\"color:var(--shiki-foreground)\">);\u003C\u002Fspan>\u003C\u002Fspan>\n\u003Cspan class=\"line\">\u003Cspan style=\"color:var(--shiki-token-keyword)\">CREATE\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-token-keyword)\"> INDEX\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-token-function)\"> ON\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-foreground)\"> ipscope_blocks \u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-token-keyword)\">USING\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-foreground)\"> gist (network inet_ops);\u003C\u002Fspan>\u003C\u002Fspan>\n\u003Cspan class=\"line\">\u003C\u002Fspan>\n\u003Cspan class=\"line\">\u003Cspan style=\"color:var(--shiki-token-keyword)\">SELECT\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-foreground)\"> country_code, asn, is_vpn\u003C\u002Fspan>\u003C\u002Fspan>\n\u003Cspan class=\"line\">\u003Cspan style=\"color:var(--shiki-token-keyword)\">FROM\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-foreground)\"> ipscope_blocks\u003C\u002Fspan>\u003C\u002Fspan>\n\u003Cspan class=\"line\">\u003Cspan style=\"color:var(--shiki-token-keyword)\">WHERE\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-foreground)\"> network \u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-token-keyword)\">>>=\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-token-keyword)\"> inet\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-token-string-expression)\"> '198.51.100.7'\u003C\u002Fspan>\u003C\u002Fspan>\n\u003Cspan class=\"line\">\u003Cspan style=\"color:var(--shiki-token-keyword)\">ORDER BY\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-foreground)\"> masklen(network) \u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-token-keyword)\">DESC\u003C\u002Fspan>\u003C\u002Fspan>\n\u003Cspan class=\"line\">\u003Cspan style=\"color:var(--shiki-token-keyword)\">LIMIT\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-token-constant)\"> 1\u003C\u002Fspan>\u003Cspan style=\"color:var(--shiki-foreground)\">;\u003C\u002Fspan>\u003C\u002Fspan>\u003C\u002Fcode>\u003C\u002Fpre>",[70,71,72],{"id":39,"text":38,"level":37},{"id":44,"text":43,"level":37},{"id":57,"text":56,"level":37},1793226505759]