From 46a1772a0b4e343d6981105e5380acac75a23742 Mon Sep 17 00:00:00 2001 From: Tommy Parnell Date: Wed, 4 Nov 2026 15:13:20 -0500 Subject: [PATCH] shuffle smaller words better --- scripts/analyze-word-variety.mjs | 44 ++++++++++++++++++++ src/engine/generator.js | 71 +++++++++++++++++++++++--------- src/engine/random.js | 18 ++++++++ src/engine/solver.js | 21 ++++++++-- src/engine/word-cap.js | 18 ++++++-- src/version.json | 2 +- 6 files changed, 148 insertions(+), 26 deletions(-) create mode 100644 scripts/analyze-word-variety.mjs create mode 100644 src/engine/random.js diff --git a/scripts/analyze-word-variety.mjs b/scripts/analyze-word-variety.mjs new file mode 100644 index 0000000..210fe39 --- /dev/null +++ b/scripts/analyze-word-variety.mjs @@ -0,0 +1,44 @@ +// Reports how often short (1-2 tile) words repeat across generated puzzles. +// Usage: node scripts/analyze-word-variety.mjs [days=365] +import { generateDailyPuzzle } from '../src/engine/generator.js'; + +const days = Number(process.argv[2]) || 365; +const counts = new Map(); +const targetCounts = new Map(); +let totalSmall = 0; +const start = new Date(2026, 0, 1); + +for (let i = 0; i < days; i++) { + const d = new Date(start); + d.setDate(d.getDate() + i); + const dateStr = `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`; + const puzzle = generateDailyPuzzle(dateStr); + for (const q of puzzle.targetQuartiles) targetCounts.set(q, (targetCounts.get(q) || 0) + 1); + for (const w of puzzle.words) { + if (w.tileCount > 2) continue; + totalSmall++; + counts.set(w.word, (counts.get(w.word) || 0) + 1); + } +} + +const sorted = [...counts.entries()].sort((a, b) => b[1] - a[1]); +const top20 = sorted.slice(0, 20).reduce((s, [, c]) => s + c, 0); +console.log(`Puzzles: ${days}`); +const maxTarget = Math.max(...targetCounts.values()); +console.log(`Target words: ${targetCounts.size} distinct, most repeated used ${maxTarget}x`); +console.log(`Short-word slots: ${totalSmall} (${(totalSmall / days).toFixed(1)}/puzzle)`); +console.log(`Distinct short words: ${counts.size}`); +console.log(`Top-20 words fill ${((top20 / totalSmall) * 100).toFixed(1)}% of short-word slots`); +const byLen = new Map(); +for (const [w, c] of counts) { + const k = w.length <= 3 ? `${w.length} letters` : '4+ letters'; + const e = byLen.get(k) || { slots: 0, distinct: 0 }; + e.slots += c; + e.distinct++; + byLen.set(k, e); +} +for (const [k, e] of [...byLen].sort()) { + console.log(` ${k.padEnd(11)} ${(e.slots / days).toFixed(1)}/puzzle, ${e.distinct} distinct, avg repeat ${(e.slots / e.distinct).toFixed(1)}x`); +} +console.log('Most repeated:'); +console.log(sorted.slice(0, 25).map(([w, c]) => ` ${w.padEnd(12)} ${c}`).join('\n')); diff --git a/src/engine/generator.js b/src/engine/generator.js index 78a13fe..a0cc8f5 100644 --- a/src/engine/generator.js +++ b/src/engine/generator.js @@ -1,28 +1,22 @@ import { TARGET_WORDS_POOL } from '../data/puzzles.js'; import { solveBoard } from './solver.js'; +import { DICTIONARY_SET } from '../data/lexicon.js'; +import { createSeededRandom } from './random.js'; + +export { createSeededRandom }; // Variety limits per board, so puzzles don't feel samey const MAX_SUFFIX_WORDS = 1; // words ending in -tion / -sion const MAX_LONG_WORDS = 2; // words with 11+ letters const LONG_WORD_LENGTH = 11; const SUFFIX_RE = /(tion|sion)$/; +// Tiles that are short words on their own ("in", "me", "per") always show up +// as single-tile answers, so a few of them would repeat on most boards. +const MAX_SHORT_WORD_TILES = 1; +const SHORT_WORD_TILE_LENGTH = 3; -/** Deterministic PRNG (mulberry32) seeded from a string via FNV-1a */ -export function createSeededRandom(seed) { - let h = 0x811c9dc5; - const str = String(seed); - for (let i = 0; i < str.length; i++) { - h ^= str.charCodeAt(i); - h = Math.imul(h, 0x01000193); - } - let a = h >>> 0; - return () => { - a = (a + 0x6d2b79f5) >>> 0; - let t = a; - t = Math.imul(t ^ (t >>> 15), t | 1); - t ^= t + Math.imul(t ^ (t >>> 7), t | 61); - return ((t ^ (t >>> 14)) >>> 0) / 4294967296; - }; +function countShortWordTiles(tiles) { + return tiles.filter(t => t.length <= SHORT_WORD_TILE_LENGTH && DICTIONARY_SET.has(t.toLowerCase())).length; } function shuffle(arr, rng) { @@ -34,6 +28,40 @@ function shuffle(arr, rng) { return a; } +// Very common tiles ("te", "ra", "er", "re") keep pairing into the same small +// words ("rate", "tire", "rest") on board after board. Each target word gets a +// selection weight that drops the more of these common tiles it contains, so +// boards lean toward a fresher mix of tiles. 0 = plain uniform shuffle. +const TILE_RARITY_STRENGTH = 0.6; + +const tileFrequency = new Map(); +for (const item of TARGET_WORDS_POOL) { + for (const t of item.tiles || []) { + const key = t.toLowerCase(); + tileFrequency.set(key, (tileFrequency.get(key) || 0) + 1); + } +} + +const weightCache = new WeakMap(); +function tileRarityWeight(item) { + let w = weightCache.get(item); + if (w === undefined) { + // Geometric mean of 1/frequency across the word's tiles + const logSum = (item.tiles || []).reduce((s, t) => s + Math.log(tileFrequency.get(t.toLowerCase()) || 1), 0); + w = Math.exp((-TILE_RARITY_STRENGTH * logSum) / Math.max(1, item.tiles?.length || 1)); + weightCache.set(item, w); + } + return w; +} + +/** Weighted random order (Efraimidis-Spirakis): rarer-tile words tend to come first */ +function weightedShuffle(arr, rng) { + return arr + .map(item => ({ item, key: Math.log(rng() || Number.MIN_VALUE) / tileRarityWeight(item) })) + .sort((a, b) => b.key - a.key) + .map(e => e.item); +} + /** * Greedily picks 5 words with 20 unique tiles from a shuffled pool while * respecting the variety limits. Returns null if it can't fill the board. @@ -43,6 +71,7 @@ function pickWords(pool, { enforceVariety = true } = {}) { const usedTiles = new Set(); let suffixCount = 0; let longCount = 0; + let shortWordTileCount = 0; for (const item of pool) { if (!item.tiles || item.tiles.length !== 4) continue; @@ -51,15 +80,18 @@ function pickWords(pool, { enforceVariety = true } = {}) { const isSuffix = SUFFIX_RE.test(item.word); const isLong = item.word.length >= LONG_WORD_LENGTH; + const shortWordTiles = countShortWordTiles(item.tiles); if (enforceVariety) { if (isSuffix && suffixCount >= MAX_SUFFIX_WORDS) continue; if (isLong && longCount >= MAX_LONG_WORDS) continue; + if (shortWordTileCount + shortWordTiles > MAX_SHORT_WORD_TILES) continue; } picked.push(item); item.tiles.forEach(t => usedTiles.add(t.toLowerCase())); if (isSuffix) suffixCount++; if (isLong) longCount++; + shortWordTileCount += shortWordTiles; if (picked.length === 5) return picked; } return null; @@ -67,8 +99,8 @@ function pickWords(pool, { enforceVariety = true } = {}) { function selectWords(candidatePool, rng) { const attempts = [ - () => pickWords(shuffle(candidatePool, rng)), - () => pickWords(shuffle(TARGET_WORDS_POOL, rng)), + () => pickWords(weightedShuffle(candidatePool, rng)), + () => pickWords(weightedShuffle(TARGET_WORDS_POOL, rng)), () => pickWords(shuffle(TARGET_WORDS_POOL, rng)), () => pickWords(shuffle(TARGET_WORDS_POOL, rng), { enforceVariety: false }) ]; @@ -110,7 +142,8 @@ function themeLabel(selectedWords) { * Guarantees: * - 20 strictly unique tiles (no duplicates anywhere on board) * - All tiles are at least 2 characters (never 1 letter) - * - Variety: at most one -tion/-sion answer and two 11+ letter answers + * - Variety: at most one -tion/-sion answer, two 11+ letter answers, and one + * tile that is a short word by itself (e.g. "in", "per") * - Anti-repetition: tracks recently used words in localStorage across games * (skipped when a seed is given, so seeded puzzles stay deterministic) */ diff --git a/src/engine/random.js b/src/engine/random.js new file mode 100644 index 0000000..b1edac3 --- /dev/null +++ b/src/engine/random.js @@ -0,0 +1,18 @@ +/** Deterministic PRNG (mulberry32) seeded from a string via FNV-1a */ +export function createSeededRandom(seed) { + let h = 0x811c9dc5; + const str = String(seed); + for (let i = 0; i < str.length; i++) { + h ^= str.charCodeAt(i); + h = Math.imul(h, 0x01000193); + } + let a = h >>> 0; + return () => { + a = (a + 0x6d2b79f5) >>> 0; + let t = a; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + diff --git a/src/engine/solver.js b/src/engine/solver.js index e318a5c..83d3d5f 100644 --- a/src/engine/solver.js +++ b/src/engine/solver.js @@ -1,5 +1,6 @@ import { DICTIONARY_SET, PREFIX_SET, getCommonnessScore } from '../data/lexicon.js'; -import { capWordList, compareCandidates } from './word-cap.js'; +import { capWordList, compareCandidates, SMALL_WORD_FUZZ, SMALL_WORD_MAX_TILES } from './word-cap.js'; +import { createSeededRandom } from './random.js'; import { POINTS_BY_TILE_COUNT, QUARTILE_COMPLETION_BONUS, EXPERT_RANK_POINTS, RANK_TIERS, getRankInfo } from './constants.js'; export { POINTS_BY_TILE_COUNT, QUARTILE_COMPLETION_BONUS, EXPERT_RANK_POINTS, RANK_TIERS, getRankInfo }; @@ -68,8 +69,22 @@ export function solveBoard(quartets, maxWords = 30) { // Rank candidates: target quartiles first, then familiar words, then tile // count (see compareCandidates). Shorter words win ties so the list isn't - // dominated by the longest possible combos. - words.sort((a, b) => compareCandidates(a, b, getCommonnessScore)); + // dominated by the longest possible combos. Small 1-2 tile words get a + // per-board random nudge so the same common short words don't win every + // board. It's seeded from the tiles, so a given board always yields the + // same list (daily and shared puzzles stay identical for everyone). + const boardSeed = tiles.join('|'); + const jitterCache = new Map(); + const getJitter = (w) => { + if (w.tileCount > SMALL_WORD_MAX_TILES) return 0; + let j = jitterCache.get(w.word); + if (j === undefined) { + j = createSeededRandom(`${boardSeed}:${w.word}`)() * SMALL_WORD_FUZZ; + jitterCache.set(w.word, j); + } + return j; + }; + words.sort((a, b) => compareCandidates(a, b, getCommonnessScore, getJitter)); // Cap at maxWords (default: 30) while keeping a fair share of 2-tile words. // Valid words beyond the cap are still accepted in play as bonus words. diff --git a/src/engine/word-cap.js b/src/engine/word-cap.js index cb2671b..adb8a79 100644 --- a/src/engine/word-cap.js +++ b/src/engine/word-cap.js @@ -1,22 +1,34 @@ +/** + * How far (in commonness tiers) small 1-2 tile words can be randomly nudged + * when ranking. Without this the same handful of very common short words + * ("rate", "fire", "late") would win every board they can be built on. The + * familiarity gate (commonness >= 2) still uses the raw score, so obscure + * words never get promoted past everyday ones. + */ +export const SMALL_WORD_FUZZ = 1.5; +export const SMALL_WORD_MAX_TILES = 2; + /** * Ranking for solver candidates (used by src/engine/solver.js). * * 1. Target quartiles first. * 2. Familiar words (commonness >= 2) before rare ones, so long obscure * combos can't crowd out everyday words. - * 3. Higher commonness tier. + * 3. Higher commonness tier (plus optional jitter from `getJitter`). * 4. More tiles (worth more points). * 5. Shorter word (avoids a bias toward the longest possible combos). * 6. Alphabetical. */ -export function compareCandidates(a, b, getCommonness) { +export function compareCandidates(a, b, getCommonness, getJitter = () => 0) { if (a.isTargetQuartile !== b.isTargetQuartile) return a.isTargetQuartile ? -1 : 1; const ca = getCommonness(a.word); const cb = getCommonness(b.word); const fa = ca >= 2 ? 1 : 0; const fb = cb >= 2 ? 1 : 0; if (fa !== fb) return fb - fa; - if (ca !== cb) return cb - ca; + const sa = ca + getJitter(a); + const sb = cb + getJitter(b); + if (sa !== sb) return sb - sa; if (a.tileCount !== b.tileCount) return b.tileCount - a.tileCount; if (a.word.length !== b.word.length) return a.word.length - b.word.length; return a.word.localeCompare(b.word); diff --git a/src/version.json b/src/version.json index e37bf6f..e025fe3 100644 --- a/src/version.json +++ b/src/version.json @@ -1,3 +1,3 @@ { - "version": "1.0.5" + "version": "1.0.6" }