{ "app": { "title": "MooresLab Verification Benchmark" }, "meta": { "title": "{{page}} · MooresLab Benchmark", "siteName": "MooresLab Verification Collateral Benchmark", "samples": "Sample library", "leaderboard": "Leaderboard", "howItWorks": "How it works", "optIn": "Get updates", "terms": "Terms of service", "privacy": "Privacy policy", "sampleLeaderboard": "{{sample}} leaderboard", "scorecard": "{{contestant}} on {{sample}}", "scorecardUnnamed": "An entry on {{sample}}" }, "common": { "loading": "Loading...", "somethingWentWrong": "Something went wrong. Try again.", "uploadLostConnection": "The upload lost its connection before we could answer, so it may not have arrived. Check your internet connection, then send it again.", "email": "Email", "password": "Password", "accountMenu": "Account menu", "signOut": "Sign out", "save": "Save", "cancel": "Cancel", "edit": "Edit", "delete": "Delete", "actions": "Actions", "yes": "Yes", "no": "No", "breadcrumbs": "Breadcrumb", "aboutPage": "About {{title}}", "clear": "Clear", "pageOf": "Page {{page}} of {{totalPages}}", "add": "Add", "remove": "Remove", "failed": { "body": "The request failed, so this page cannot show what is here. That is a fault at our end, not an empty result. Try again in a moment.", "stale": "The last refresh failed, so what you see here may be out of date.", "retry": "Try again" }, "done": "Done", "search-placeholder": "Search", "search-no-results": "Nothing matches that search.", "columns": "Columns", "select": "Select", "select-all": "Select all", "move": "Move" }, "unsavedChanges": { "title": "Leave without saving?", "body": "You have changes on this page that are not saved yet. If you leave now, they are lost.", "stay": "Stay on this page", "leave": "Leave without saving" }, "table": { "results_one": "{{count}} row", "results_other": "{{count}} rows", "rows-per-page": "Rows per page", "pagination": "Pages", "sort": "Sort", "sort-ascending": "Sorted ascending", "sort-descending": "Sorted descending" }, "auth": { "signIn": { "title": "Welcome back", "subtitle": "Sign in to your account.", "action": "Sign in", "next": "Next", "changeEmail": "Change", "noAccount": "New here?", "goToSignUp": "Create an account", "forgotPassword": "Forgot your password?", "invalidCredentials": "Invalid login credentials." }, "oauth": { "continueWith": "Continue with {{provider}}", "errors": { "unavailable": "That sign-in provider is not available right now. Use your email and password.", "denied": "The sign-in was cancelled. Nothing has changed.", "expired": "That sign-in took too long. Start again.", "emailUnavailable": "That provider shared no email address, so there is no account to sign in to.", "emailUnverified": "That provider has not verified your email address, so we cannot sign you in.", "failed": "That sign-in could not be completed. Try again." } }, "signUp": { "title": "Create your account", "subtitle": "A minute from now, you're in.", "action": "Create account", "haveAccount": "Already have an account?", "goToSignIn": "Sign in", "accept": "I have read and accept the Terms of service and the Privacy policy.", "acceptRequired": "Accept the terms of service and the privacy policy to carry on.", "productNews": "Email me product news and launch updates. Optional, and every email has an unsubscribe link." }, "forgotPassword": { "title": "Reset your password", "subtitle": "Enter your email and we'll send a reset link.", "subtitleKnown": "We'll send a reset link to {{email}}.", "action": "Send reset link", "backToSignIn": "Back to sign in" }, "resetPassword": { "title": "Choose a new password", "subtitle": "The link is valid for 30 minutes.", "newPassword": "New password", "action": "Set new password", "missingToken": "This reset link is incomplete. Request a new one." }, "verifyEmail": { "title": "Email verification", "verifying": "Verifying your email address...", "verified": "Your email address is verified. Welcome!", "failed": "That link is invalid or has expired.", "resend": "Send a new verification email", "resent": "Check your inbox for a fresh link.", "goToDashboard": "Go to the dashboard" }, "changeEmail": { "title": "New email address", "confirming": "Confirming your new address...", "confirmed": "Done. Your account now signs in with {{email}}.", "failed": "That link is invalid or has expired. Request a new one from your security settings.", "goToDashboard": "Go to the dashboard" }, "emailCode": { "switchBack": "Use your password instead", "requestAction": "Send me a code", "notSent": "We could not send a code to {{email}} yet. Ask again once the wait is over.", "sent": "We sent a code to {{email}}. It expires in 10 minutes.", "codeLabel": "Sign-in code", "verifyAction": "Sign in" }, "mfa": { "title": "One more step", "subtitle": "This account is protected by two-factor authentication.", "body": "Enter the code from your authenticator app.", "codeLabel": "Authentication code", "action": "Verify", "useRecovery": "Lost your phone? Use a recovery code", "useApp": "Use your authenticator app instead", "recoveryBody": "Enter one of the recovery codes you saved when you set up two-factor. Each one works once.", "recoveryLabel": "Recovery code" }, "passkey": { "action": "Sign in with a passkey", "failed": "That passkey sign-in did not complete. Try again, or use your password." }, "validation": { "emailInvalid": "Enter a valid email address.", "passwordTooShort": "Passwords must be at least {{count}} characters.", "fillAllFields": "Fill in every field to continue." }, "rateLimited_one": "Too many attempts. Try again in {{count}}s", "rateLimited_other": "Too many attempts. Try again in {{count}}s", "acceptInvitation": { "title": "Accept your invitation", "subtitle": "Choose a password for {{email}}.", "password": "Password", "action": "Create my account", "loading": "Reading your invitation", "invalid": { "title": "This invitation can no longer be used", "subtitle": "The link has expired, was already used, or was withdrawn. Ask whoever invited you to send it again, or sign in if you already have an account.", "signIn": "Go to sign in" }, "failed": { "title": "We could not read this invitation" } }, "chooseNewPassword": { "title": "Choose a new password", "subtitle": "An operator reset your password. Choose one of your own before you carry on.", "temporary": "Temporary password", "action": "Set my password" } }, "connect": { "consent": { "title": "Connect {{client}}", "titleUnknown": "Connect an AI agent", "subtitle": "{{client}} is asking to act as you on the benchmark.", "verifiedBy": "Published by {{host}}, which vouches for this agent's name.", "unverified": "This agent registered itself, so nobody vouches for its name. Only continue if you started this connection yourself.", "redirect": "After you decide, you go back to {{host}}.", "loopbackWarning": "That address is a program on this computer. Continue only if you just ran a command that asked to connect.", "permissions": "If you allow it, it will be able to:", "identityOnly": "Know who you are. It asks for no other permission.", "signedInAs": "You are signed in as {{email}}. It will act as this account, never with more than you can do yourself.", "approve": "Allow", "deny": "Deny", "expired": "This request expired or was already answered. Start the connection again from your agent.", "passwordChangeRequired": "Your account is on a temporary password. Choose a new one before you connect an agent.", "choosePassword": "Choose a new password" }, "errors": { "title": "This connection cannot continue", "invalidClient": "The agent that sent you here is not one this site recognizes.", "invalidRedirectUri": "The agent asked to send you somewhere it never registered, so this site stopped it.", "invalidRequest": "The agent's request was incomplete.", "serverError": "Something went wrong on our side. Try connecting again.", "unknown": "The agent's request could not be completed. Try connecting again from your agent." } }, "theme": { "label": "Theme", "current": "Theme: {{choice}}", "currentChoice": "{{choice}}, current theme", "choices": { "light": "Light", "dark": "Dark", "system": "System" } }, "languagePicker": { "label": "Language", "current": "Language: {{language}}" }, "settings": { "navLink": "Settings", "title": "Settings", "profile": { "tab": "Profile", "detailsTitle": "Your details", "firstName": "First name", "lastName": "Last name", "timeZone": "Time zone", "timeZoneHelp": "Dates and times are shown in this zone.", "locale": "Application language", "saved": "Saved.", "avatarTitle": "Profile picture", "avatarHelp": "JPEG, PNG, WebP, or GIF, up to 5 MB. Choose one, or drop it anywhere on this page, and it is saved at once. We crop it square and resize it to 512 pixels.", "avatarChoose": "Choose an image", "avatarRemove": "Remove", "avatarTooLarge": "That image is larger than 5 MB. Choose a smaller one.", "nameRequired": "Enter a name. It is how the leaderboard shows you.", "nameTooLong_one": "At most {{count}} character.", "nameTooLong_other": "At most {{count}} characters.", "avatarVisibility": "Your picture appears on your public page only while it is not private. You choose that under Visibility below.", "avatarNotImage": "That file is not a JPEG, PNG, WebP or GIF image. Choose one of those.", "avatarDropTitle": "Drop your picture here", "avatarDropHint": "It replaces your profile picture as soon as you let go.", "avatarDropOne": "Drop one image at a time.", "avatarDropFolder": "That is a folder. Drop one image file.", "avatarDropBusy": "Your picture is still changing. Wait for it to finish, then try again.", "visibilityTitle": "Visibility", "visibilityLede": "Your first and last name are always public. These two choices decide what else strangers can see." }, "security": { "tab": "Security", "password": { "title": "Password", "subtitle": "Changing your password signs out every other browser.", "current": "Current password", "new": "New password", "action": "Change password", "changed": "Your password is changed. Other sessions were signed out.", "confirm": "Confirm new password", "mismatch": "The two passwords don't match." }, "twoFactor": { "title": "Two-factor authentication", "subtitle": "An authenticator app code, asked for every time you sign in with a password.", "enabled": "On", "disabled": "Off", "unknown": "Unknown", "unreadable": "We could not read this setting, so it is shown as neither on nor off.", "enableAction": "Set up two-factor", "disableAction": "Turn off", "enrollTitle": "Set up two-factor authentication", "enrollBody": "Scan this code with your authenticator app, then enter the code it shows.", "qrAlt": "QR code for your authenticator app", "manualEntry": "Can't scan? Enter this secret by hand:", "codeLabel": "Code from your app", "codeHelp": "Six digits, refreshed every 30 seconds.", "confirmAction": "Turn on two-factor", "recoveryTitle": "Save your recovery codes", "recoveryBody": "Each code signs you in once if you lose your authenticator. This is the only time they are shown.", "recoveryCopy": "Copy the codes", "recoveryCopied": "Copied", "recoveryDone": "I've saved them" }, "passkeys": { "title": "Passkeys", "subtitle": "Sign in with your device or password manager, with no password to type.", "name": "Name", "added": "Added", "lastUsed": "Last used", "neverUsed": "Never", "empty": { "title": "No passkeys yet", "body": "A passkey signs you in with this device, and there is no password to type." }, "failed": { "title": "We could not load your passkeys" }, "nameLabel": "Name this passkey", "nameHelp": "Something you'll recognize, such as the device it lives on.", "addAction": "Add a passkey", "ceremonyFailed": "That passkey was not registered. Try again.", "unsupported": "This browser cannot create passkeys." }, "sessions": { "title": "Active sessions", "subtitle": "Every browser currently signed in to this account.", "signedIn": "Signed in", "expires": "Expires", "current": "This browser", "revoke": "Sign out", "empty": { "title": "No sessions to show", "body": "This browser's own session belongs here. Ask again if it is missing." }, "failed": { "title": "We could not load your sessions" } }, "connectAgent": { "title": "Connect an AI agent", "subtitle": "Let Claude Code, Codex or another agent browse the samples, read the leaderboard and your scorecards, and enter for you. Add this site to your agent with the command below.", "agents": { "claudeCode": "Claude Code", "codex": "Codex" }, "copy": "Copy", "copied": "Copied", "signIn": "The first time your agent uses the site, it opens your browser to sign in and asks you what it may do: read, enter, or both.", "entering": "Entering needs an agent that can upload a file, such as Claude Code or Codex. It asks for a one-time link and sends your zip archive to it, under the same rules and the same five a day as the upload form here.", "otherClients": "Any other agent that speaks MCP over HTTP connects to {{url}}. One that cannot upload a file can read the benchmark but not enter it." }, "connectedApps": { "title": "Connected apps", "subtitle": "AI agents you allowed to act as you. Revoking one ends its access, and any upload link it holds, at once.", "app": "App", "permissions": "May", "lastUsed": "Last used", "verifiedBy": "Published by {{host}}", "unverified": "Registered itself, so nobody vouches for its name", "connected": "Connected {{date}}", "identityOnly": "Know who you are", "revoke": "Revoke", "scopes": { "read": "Read", "enter": "Enter", "unknown": "A permission this site no longer describes." }, "empty": { "title": "No apps are connected", "body": "Nothing acts as you here but you. Connect an agent with the commands above." }, "failed": { "title": "We could not load your connected apps" } }, "danger": { "title": "Delete this account", "subtitle": "Your account, and everything only you own, is deleted immediately. This cannot be undone.", "action": "Delete account", "modalTitle": "Delete this account?", "modalBody": "This deletes your account and everything only you own. It cannot be undone.", "confirmLabel": "Type {{email}} to confirm", "confirmAction": "Delete my account", "zoneTitle": "Danger zone" } }, "sessions": { "tab": "Sessions" }, "mcp": { "tab": "MCP" }, "email": { "title": "Email address", "verified": "Verified", "unverified": "Not verified", "newAddress": "New email address", "help": "We'll email a confirmation link there. Nothing changes until you open it.", "action": "Send confirmation link", "sent": "Check {{email}} for the confirmation link.", "confirmTitle": "Confirm it's you", "confirmBody": "Enter your password to send a confirmation link to {{email}}." } }, "notifications": { "bell": "Notifications", "heading": "Notifications", "empty": "Nothing here yet.", "markAllRead": "Mark all read" }, "nav": { "productName": "Benchmark", "samples": "Samples", "leaderboard": "Leaderboard", "methodology": "How it works", "submissions": "My entries", "enter": "Enter", "accountSettings": "Account settings", "signIn": "Sign in", "register": "Register", "signedInAs": "Signed in as", "adminSettings": "Admin settings", "feedback": "Send feedback", "help": "Help and feedback" }, "brand": { "website": "MooresLab website (opens in a new tab)", "websiteHint": "Visit mooreslab.ai", "home": "Benchmark home", "homeHint": "Back to the benchmark's home page" }, "footer": { "tagline": "The public benchmark for verification collateral. Enter your testbench and have it rated against a published methodology.", "benchmark": "Benchmark", "legal": "Legal", "getUpdates": "Get updates", "copyright": "© {{year}} MooresLab AI", "disclaimer": "Ratings are produced by an AI rating system. See the methodology for its error margin." }, "legal": { "eyebrow": "Legal", "version": "Version {{version}}", "versionUnavailable": "The version of this document could not be read just now. The wording below is what this deployment is serving.", "accept": { "title": "Please read these before carrying on", "lede": "The terms of service and the privacy policy have changed, or you have not agreed to them yet. Read both, then accept them here. Accepting records which version you agreed to and when.", "action": "I accept both", "failed": "That could not be recorded just now. Try again." }, "contact": { "title": "How to reach us", "email": "privacy@mooreslab.ai", "body": "Write to {{email}}. Say which of these two documents you are writing about, and we will answer." }, "terms": { "title": "Terms of service", "lede": "These terms cover the MooresLab Verification Collateral Benchmark at benchmark.mooreslab.ai. Holding an account here means agreeing to them.", "sections": [ { "title": "Who may enter", "body": "Anybody may register with any working email address, or be invited by whoever operates this deployment, and anybody with an account may enter. You must be at least 16 years old and able to agree to these terms on your own behalf.", "points": [ "Your email address is verified, by a one-time code or by the invitation link that brought you here. Nothing else you tell us is verified.", "Keep your password to yourself. Anything done with your account is treated as done by you.", "If you are locked out, whoever operates this deployment can reset your account to a temporary password, given to you once and never stored. Every session your account holds ends, and you must choose your own password the next time you sign in before you can do anything else.", "One person, one account. Entering the same collateral from two accounts is what the duplicate rule below exists to stop.", "You may delete your account at any time, from your account settings." ] }, { "title": "What the benchmark does", "body": "You download a sample: an IP block with its specification and its RTL. You build the verification collateral you would ship for it, and you upload one archive holding all of it. An automated rater reads the archive and scores it against a rubric published in full before anybody enters.", "points": [ "Six dimensions are rated: the test plan, the drivers and monitors, the scoreboard and predictor, the assertions, the functional coverage model, and the test suite.", "The specification is not rated. It ships with the sample, so it is not your work.", "Coverage reports are not accepted as evidence. Only source is rated, because a reported percentage is trivial to fabricate and impossible to verify.", "How you produced the collateral is neither asked nor rated. Our tools, somebody else's, or a person and a text editor all enter the same way." ] }, { "title": "What you upload, and what you are promising about it", "body": "You keep ownership of everything you upload. What you are promising is that it is yours to upload.", "points": [ "You have the right to enter the collateral, and entering it breaks no agreement you are under.", "It carries no employer confidential material, no third party's material held under a non-disclosure agreement, and nothing subject to export control.", "It carries no personal information about anybody other than yourself.", "It carries no malicious code and nothing intended to interfere with the rating system or with this site." ] }, { "title": "What we may do with it", "body": "You grant MooresLab AI a non-exclusive licence to store your archive, to have an automated rater read and score it, and to read it ourselves for the three purposes below. The licence covers nothing else. We do not publish your collateral, sell it, show it to another contestant, or use it to train a model.", "points": [ "Rating it, which means sending its contents to the third-party model the rating runs on. Which vendor and which model rated an entry is published beside the score.", "Screening it before it is rated, by sending a short digest of it to a smaller model from the same vendor. An entry the screen flags is held until an operator reviews it.", "Reviewing how the rubric performed against it, which is one of the two reasons this benchmark exists and is stated here rather than buried.", "Investigating an appeal, a suspected duplicate entry, or a report of abuse.", "The licence over the archive ends when the archive is deleted, which is within 90 days. The score it produced stays published while the benchmark runs." ] }, { "title": "What gets published", "body": "A rating produces a public row and a scorecard with two halves. The row and the public half of the scorecard show how your entry scored. The private half shows why, citing your collateral, and no other contestant ever sees it.", "points": [ "A public row carries the sample, your first and last name, your score, and when you entered it.", "The public half of your scorecard carries the score of every dimension and every criterion: its weight, its mark, and whether it applied. It names no file of yours and quotes nothing you wrote.", "The private half carries the evidence, the findings, and the files you uploaded. Only you and whoever operates this deployment can open it. The raters' full reports are opened only by whoever operates this deployment.", "Neither ever carries your email address. Both carry your account id, which is how your name links to your page, and your organization appears only when you turn on showing it in your account settings.", "Your best entry per sample is the one that counts, so re-entering can raise your place and never lowers it." ] }, { "title": "Your public page", "body": "Every contestant has a page at /u/ followed by their account id. It is private until you make it public in your account settings, and you can make it private again at any time. While it is public it shows your picture, your bio, your links and your ranked entries to anybody.", "points": [ "What you put on it is yours to publish, and none of it pretends to be somebody else.", "It carries nothing unlawful, abusive, sexually explicit or misleading, and no link to malicious software or to a page that tries to collect somebody's password.", "It carries no personal information about anybody other than yourself.", "Breaking these rules is grounds for whoever operates this deployment to remove your bio, your links or your picture, and you are shown why in your account settings. It is also grounds for suspension, which takes your page down along with your rows on the leaderboard." ] }, { "title": "Comments and reactions", "body": "A scorecard anybody may read can carry a comment section, which we may offer, pause or withdraw. Where one is offered, once your account is set up, your address is verified and you have accepted these terms, you may comment on it, reply to a comment, like or dislike one, and leave one of five reactions on the scorecard.", "points": [ "A comment and a reaction are public, signed with your first and last name, and with your organization when you show it.", "Do not post abuse, threats, slurs, spam, anybody else's private information, or collateral that is not yours to share.", "A comment that matches our list of abusive words is hidden from everybody as soon as it is written, a comment more people dislike than like is hidden too, and whoever operates this deployment may take any comment down.", "Anybody signed in may report a scorecard, a comment or a contestant's public page, with a written reason, and whoever operates this deployment reviews every report.", "You may delete your own comments, and nobody can edit one. A deleted comment that has replies stays as \"Comment removed\" so the conversation under it still reads.", "You may write up to thirty comments and replies an hour." ] }, { "title": "The rating, and the one appeal", "body": "Every pass is independent, and runs under the model and the skills version this deployment is pinned to. That pin is what makes two scores comparable. Rating is not deterministic: the same collateral entered twice may not score identically, which the pin bounds rather than removes.", "points": [ "You may ask for one re-grade per entry, carrying the argument for it. The higher of the two scores stands.", "A pass that failed on our side is not a rating. It is not shown as a score, it costs you no appeal, and we re-run it.", "The rating is one-way. There is no forum, no comment thread, and no public debate about a score." ] }, { "title": "Fair play", "body": "Entering is meant to be a test of verification collateral. These are the things that would stop it being one.", "points": [ "The same archive cannot be entered twice, by you or by anybody else.", "Do not enter work that is not yours to enter.", "Do not try to instruct, mislead or manipulate the rater through the contents of your archive or your notes. Notes are read as context and can never change a score.", "Do not attack this deployment, probe it without asking us first, or try to reach another contestant's collateral.", "You may enter five times a day." ] }, { "title": "Connected AI agents", "body": "You may connect an AI agent, such as Claude Code or Codex, to your account. It then acts as you, and anything it does here is treated as done by you.", "points": [ "It can do only what you allowed on the screen that asked you. Reading lets it browse the samples, download a sample's files, read the leaderboard and scorecards as you can, and see your own entries. Entering lets it start an entry in your name.", "An entry an agent makes is your entry. It is held to these terms, counts toward your five a day, and is rated, published and appealed exactly as one you uploaded yourself.", "To enter, the agent is given a link that works once, for five minutes, and uploads your archive to it. An agent that cannot upload a file can read but not enter.", "An agent never acts with an operator's powers, even when the account that connected it has them.", "You can revoke a connection at any time from the MCP section of your account settings, and its access and any upload link it holds end at once. Resetting your password, or an operator setting you a temporary password, revokes every connection." ] }, { "title": "Suspension", "body": "We may suspend an account that breaks these terms. A suspended account cannot enter and cannot appeal, and its rows leave the public leaderboard. It can still sign in, still read its own entries, and is always shown the reason it was suspended.", "points": [ "We may also take down an entry, a scorecard or a comment, or remove the bio, the links or the picture from a contestant's page, that breaks these terms without suspending anybody. A taken-down entry leaves the leaderboard, and its entrant is shown why, and so is a contestant whose page we change.", "A suspension is reversible, and lifting one brings the standings back with it.", "If you believe a suspension is wrong, write to us at the address below." ] }, { "title": "What we do not promise", "body": "This is a benchmark we publish, not a service you buy. It is provided as it is, with no warranty of any kind: we do not promise that it will be available, that a rating is free of error, or that the site will work tomorrow the way it works today. To the fullest extent the law allows, MooresLab AI is not liable for any loss arising from your use of it.", "points": [ "We may change or withdraw a sample, revise the rubric, or stop running the benchmark.", "Withdrawing a sample takes its standings off the board. Nothing is destroyed by it, and republishing brings them back.", "Nothing here is a promise of employment, of a commercial relationship, or of a score." ] }, { "title": "Changes to these terms", "body": "These terms carry a version, printed at the top of this page. When the terms change, the version changes, and you are asked to read and accept the new one the next time you sign in. What you accepted, and when, is recorded against your account. If you would rather not accept a new version, you may delete your account.", "points": [] }, { "title": "Governing law", "body": "These terms are governed by the law of the United States and of the state in which MooresLab AI has its principal place of business, and any dispute arising from them belongs to the courts there.", "points": [] } ] }, "privacy": { "title": "Privacy policy", "lede": "What this site collects, what it publishes, who reads what you upload, and how long any of it is kept.", "sections": [ { "title": "Who this is about", "body": "MooresLab AI runs the Verification Collateral Benchmark at benchmark.mooreslab.ai and is responsible for the information described here. This deployment stands on its own: no other MooresLab system reads from it, and the only credentials it holds for anything else of ours can do nothing but file issues in a single repository and post to a single Slack channel.", "points": [] }, { "title": "What we collect", "body": "As little as the benchmark needs. Most of the fields below are optional, and the phone number is never published.", "points": [ "Your email address. Required, and verified by a one-time code or by the invitation link that brought you here, because it is how we reach you and how you get back into your account.", "Your password, stored only as an argon2id hash. We never see it and cannot recover it.", "An invitation, if an operator invited you: the address it went to, who sent it, whether it makes you an operator, and when it was sent, resent, accepted or withdrawn. It is kept from the moment it is sent, before any account exists, and stays after it is answered as a record of how the account was opened.", "Your first and last name. Required before you can enter, because they are what names you on the leaderboard.", "Your organization. Optional, and public only when you turn on showing it.", "Your phone number. Optional, never verified, and never published. It exists in case we need to reach you.", "Your public page, if you fill it in: a bio, up to four links, a profile picture and the country you choose, each optional, and whether the page is private, which it is until you change it.", "Your profile picture, if you add one. Before it is saved it is checked automatically by the same image model that checks uploads, on our own servers, and a sexually explicit picture, or one that cannot be checked, is not saved; when the check refuses a picture, we record that it did, and when, and never the picture. We crop the picture we keep to a square, shrink it, and save it again as a new image, which drops whatever the original file said about where and when it was taken.", "The archive you upload, the SHA-256 hash of it, and any notes you send with it.", "Sign-in and account records: your sessions, any passkey or authenticator app you enrol, and an append-only log of each sign-in and each change made to your account, carrying the IP address and the browser the request came from, and the approximate location your browser reported. To report one, your browser asks GeoJS, a third-party service, for an approximate location at sign-in; GeoJS answers from the IP address your browser connects from, and we store the city, region and country it reports. Only whoever operates this deployment can read it, and it exists to keep accounts secure.", "Whether our email reaches you. If Brevo, our email provider, tells us it has stopped delivering to your address, because you unsubscribed with the button your mail client shows, the address bounced, or you marked one of our emails as spam, we record the address, which of those it was, and when, so whoever operates this deployment can see why you are not receiving our emails. Only they read it, and we never unblock an address on our own.", "Ordinary server logs, which carry request metadata and are kept only as long as they are useful for running and securing the site.", "Error reports. When the site fails in your browser or on our servers, we record what failed, the page it failed on and the filters in its address, the internal id of the account that was signed in, and your browser's name, with email addresses, passwords and tokens removed before anything is stored. They are filed privately for our engineers and never published.", "The comments and replies you write under a scorecard, the likes and dislikes you give comments, and the reaction you leave on a scorecard.", "A report you make about a scorecard, a comment or a contestant's page, with the reason you write. Only whoever operates this deployment reads it, and the person you report is not told who you are.", "Feedback you send us: your message, the subject you give it, the entry it is about if you choose one, any file you attach, every reply in the conversation and any file attached to one, and when each was written. Only you and whoever operates this deployment read it, files included, and it is never published. Before a file is stored, every image in it is checked by the same image model that checks uploads, on our own servers, and one that is sexually explicit, or cannot be checked, is refused and not kept." ] }, { "title": "What is published, and what never is", "body": "The leaderboard is the public part of this site, and it is deliberately thin.", "points": [ "A public row carries the sample, a first and last name, a score, and when the entry was made.", "The public half of a scorecard carries the score of every dimension and criterion and nothing that quotes or names your collateral.", "It never carries an email address. It carries your account id, because your name links to your page at /u/ followed by that id.", "Your page is private until you make it public in your account settings. A private page shows your first and last name, and your organization when you show it, and nothing else.", "A public page also shows your picture, your bio, your links, your country with its flag, and the entries the leaderboard already ranks, with their places. It never shows your email address or your phone number.", "Your picture is shown only where your page is public. While it is private, only you and whoever operates this deployment can see it.", "Your organization appears only when you have turned on showing it.", "Your collateral is never published, and no other contestant ever sees it or is told what it contains.", "The private half of your scorecard, with its evidence, findings and files, is shown to you and to whoever operates this deployment, and to nobody else. The raters' full reports are shown only to whoever operates this deployment.", "A comment you write is public under the scorecard it is on, with your first and last name, and your organization when you show it. So is the reaction you leave on a scorecard, and how many people liked a comment.", "Your dislikes are never published. Nobody else, the author of the comment included, can see that you disliked it, and no comment ever shows its dislike count. Only whoever operates this deployment can see who disliked what.", "Every comment is checked against a list of abusive words as it is written, by our own server and by no third party, and a comment the list matches is hidden from everybody else automatically. Its author always sees it." ] }, { "title": "What happens to the archive you upload", "body": "This is the part worth reading twice, because it is the part where your work leaves your machine.", "points": [ "Before it is stored, every image in it is checked automatically by an image model that runs on our own servers, so no image is sent to anybody else to be checked. No sexually explicit image is accepted anywhere on this site. An upload holding one, or holding an image that cannot be checked, is refused and nothing of it is kept; we record that an upload was refused, by whom and when, and never the image. The same check covers the files whoever operates this deployment uploads to a sample.", "It is stored in a private cloud storage bucket that nothing public can read. Download links are signed and expire within minutes.", "Before it is rated, a smaller model from the same vendor reads a short digest of it: its file names, your notes, and the opening of a few text files. That check runs once. If it flags the entry as abusive or as not verification collateral, or cannot reach a verdict, the entry is held, and an operator decides whether it is rated, checked once more, or taken down.", "An automated rater reads it. The rater is an agentic command-line tool signed in to a third-party large language model vendor, so the contents of your archive are sent to that vendor's service in order to be read.", "MooresLab engineers may read an entry to investigate an appeal, to check a suspected duplicate, or to see how the rubric performed against it.", "It is never shown to another contestant, never published, never sold, and never used to train a model." ] }, { "title": "How long it is kept", "body": "An entry archive is deleted within 90 days of being uploaded, every version of it. Ninety days outlasts an appeal and any re-run we might need, and it is close to the shortest period that does. The storage that holds the archives enforces the same number.", "points": [ "The score and the scorecard an archive produced are kept while the benchmark runs. They carry findings and evidence, not your source.", "Your account and your profile are kept until you delete your account.", "Your bio, your links, your country and your picture are kept until you change or remove them, whoever operates this deployment removes one that breaks the terms of service, or you delete your account. When one is removed that way, we keep which part went, when, and the reason you are shown, and never the removed words or picture. Making your page private hides them at once, though a browser that already showed somebody your picture may keep its own copy until it clears its cache.", "Deleting your account removes your profile, your entries and their ratings, and takes your rows off the leaderboard. The uploaded archives themselves are left to the same 90-day sweep, with nothing pointing at them any more.", "A session lasts until it expires or you end it from your account settings. The log of sign-ins and account changes is kept while the benchmark runs, including after you delete your account, because a security record that vanished with the account it describes could not answer what happened to it.", "Comments, likes, dislikes and reactions are kept while the benchmark runs, and go with your account when you delete it. A comment you delete disappears from the scorecard at once, and its words are kept for whoever operates this deployment in case somebody reported it.", "A record that Brevo stopped delivering to your address is kept until whoever operates this deployment unblocks the address, which they do when you ask; it is not deleted with your account, because it describes the address rather than the account.", "Feedback conversations are kept while the benchmark runs, and go with your account when you delete it. Nothing in one is ever edited after it is written. A file attached to one, by you or by us, is deleted within 90 days of being sent; the conversation keeps its name and size.", "If your account is suspended, every comment, like, dislike and reaction you left is deleted at once and for good, and lifting the suspension does not bring them back. A comment somebody else replied under stays as \"Comment removed\", with neither your name nor your words, so their replies still read." ] }, { "title": "Cookies", "body": "One cookie, and it is the session. It is set when you sign in, scripts cannot read it, and it is not shared with anybody. There is no analytics, no advertising, and no third-party tracking anywhere on this site.", "points": [] }, { "title": "Who else touches your information", "body": "As few as we can manage, and each one is here because something has to do the job.", "points": [ "Google Cloud hosts the site, its database and its storage bucket, in the United States.", "Brevo, our email provider, sends the verification codes, sign-in codes, password reset links and invitations, and runs the two optional email lists described below. It receives your email address and the message.", "When something is reported, Brevo also sends an alert to the people who review reports, carrying the opening of the reason the reporter wrote and never the content that was reported.", "When you send feedback, Brevo sends an alert to the people who answer it, carrying your name, the opening of what you wrote and how many files you attached, never their names or what is in them. When one of them replies, Brevo sends you an email saying so, with a link to the conversation and nothing of the reply. If you answer that email by replying to it, Brevo receives your reply and passes it to us, and we add what you wrote above the email it quotes to the conversation as your message, leaving out the quoted email, your signature and any attachment; a reply from any other address than your account's is not added. Writing feedback sends you no email.", "When an entry of yours is rated, or we cannot rate it and have to run it again, Brevo sends you an email saying so, carrying the sample's name, your score and a link to your scorecard, and nothing the raters wrote.", "Slack, where the people who run this deployment work, receives a short notice in one channel they read: when you set up your account, naming your first and last name; when an entry is made, rated, fails to be rated or is deleted, naming the sample, your first and last name and, once it is rated, its score; when something is reported, saying what kind of thing and, for an automated check, its category, never the reason or what was reported; when you send feedback, naming you and never a word of what you wrote or anything you attached; and when something about how this site runs needs its operators, such as a budget or a failure of its own, which names nobody. Slack keeps those messages for as long as the workspace keeps its history. No notice carries your email address, your notes, anything from your archive, or anything else you wrote.", "The large language model vendor the raters sign in to receives the contents of the archive being rated. Which vendor and which model rated an entry is published beside the score.", "GeoJS, a third-party location service, is asked by your browser for its approximate location when you open the sign-in, sign-up or invitation page. Your browser asks it directly, so GeoJS sees the IP address your browser connects from, and nothing else of ours is sent to it.", "Nobody buys any of this from us, because none of it is for sale." ] }, { "title": "Connected AI agents", "body": "If you connect an AI agent, we keep a record of the connection, so it can act as you and so you can see and end it.", "points": [ "Which agent it is: the name it gives and, when a site vouches for it, that site. What you allowed it to do, when you connected it, and when it was last used.", "The tokens it holds, stored only as SHA-256 hashes, so nothing in our database could be presented as one. An access token lasts an hour and is renewed through a refresh token that lasts thirty days from its last use.", "Each one-time upload link it asks for, stored as a hash beside your account, the sample, your notes and the connection that asked, and never usable after five minutes or after one upload.", "Connecting an agent, revoking one, and an agent starting an upload are recorded in your account's log, naming the agent. Only whoever operates this deployment reads that log.", "Whatever your agent reads from us reaches the agent's own vendor, the same as if you had copied it there yourself. What that vendor does with it is between you and them." ] }, { "title": "The two email lists", "body": "Both are optional, both are run by Brevo, and every email sent to either carries an unsubscribe link.", "points": [ "Benchmark updates. An address entered on the Get updates page, with its box ticked, goes straight to Brevo and joins a list for news about the benchmark: new samples, contests and results. It is not an account, and this site keeps no copy of the address. All we keep is the moment the form was used, for one day, so we can limit how often it is used. Addresses that asked, before the site opened, to hear when it did are on the same list.", "Product news. An account joins a list for product news only when the box on the sign-up form is ticked, and that box arrives unticked.", "Deleting your account takes it off the product-news list, and changing your email address moves its place on that list to the new address.", "To leave either list, use the unsubscribe link in any email, or write to the address below and we will take you off it." ] }, { "title": "Where it is processed", "body": "In the United States. If you are writing from somewhere else, entering means your information is transferred there and handled under the terms set out here.", "points": [] }, { "title": "How it is protected", "body": "This deployment was built on the assumption that somebody will attack it.", "points": [ "No other system of ours reads from it. The only credential it holds for anything else of ours lets it file issues in one repository and nothing more, so if it is compromised, nothing else of ours goes with it.", "Passwords are hashed with argon2id. The credentials the raters sign in with are encrypted at rest.", "Uploads are bounded and inspected as they stream, and a refused upload leaves no entry, no queued rating and no stored file. We count refused uploads by when and why, naming nobody, and keep which account sent one for a day, only to stop one account inflating that count.", "No promise of perfect security is being made here. If we get it wrong, we would rather have held less of your work for less time, which is what the 90 days is for." ] }, { "title": "What you can ask us for", "body": "Write to the address below and we will do these things. Two of them you can also do yourself, right now, without asking.", "points": [ "A copy of what we hold about you.", "A correction to anything that is wrong. Your name, your organization, your phone number, your picture, your bio, your links, your country and whether your page is private you can change yourself in your account settings.", "Deletion of your account and everything attached to it, which you can also do yourself from your account settings.", "Withdrawal of your agreement, which means deleting the account: entering the benchmark is the only thing any of this is collected for." ] }, { "title": "Children", "body": "This site is not for anybody under 16, and we do not knowingly collect anything from one. If you believe a child has registered, write to us and we will delete the account.", "points": [] }, { "title": "Changes to this policy", "body": "This policy carries a version, printed at the top of this page. When it changes, the version changes, and you are asked to read and accept the new one the next time you sign in. What you accepted, and when, is recorded against your account. Nothing already collected is put to a new use without asking you first.", "points": [] } ] } }, "dimensions": { "testplan": { "label": "Test plan", "blurb": "Whether the plan traces to the specification, covers its features and registers, and plans for corner cases, errors and negative scenarios." }, "drvMon": { "label": "Drivers and monitors", "blurb": "Whether the agents speak the protocol correctly and completely: the right side driven, at the right time, sampled on the right edge." }, "sbPred": { "label": "Scoreboard and predictor", "blurb": "Whether the reference model matches the specification, checks everything the design promises, accounts for every transaction, and stays correct through reset." }, "sva": { "label": "Assertions", "blurb": "Whether the protocol rules are asserted correctly and completely, bound to the design, aware of reset, and never vacuously true." }, "fcov": { "label": "Functional coverage", "blurb": "The covergroups you wrote: their structure, bins, crosses and illegal bins, and that none of it is vacuous. The model is rated, never a reported percentage." }, "testcases": { "label": "Test suite", "blurb": "Whether the tests reach every feature and test plan row, check their own results, drive meaningful stimulus, and trace back to the plan." } }, "optIn": { "title": "Get updates", "about": "News about the benchmark by email: new samples, contests and results. It is not an account, and you do not need one to read anything on this site.", "lede": "Hear about new samples, contests and results by email.", "notAnAccount": "Updates only, not an account. Anybody can sign up and enter a testbench whenever they are ready.", "consent": "Email me updates about the benchmark. Every email has an unsubscribe link.", "action": "Get updates", "signUp": "or sign up now", "privacy": "Your address goes to Brevo, our email provider, and this site keeps no copy of it. Privacy policy", "invalidReason": "Enter your email address and tick the box.", "success": "You're on the list. We'll email you when there is news.", "invalid": "That does not look like an email address.", "rateLimited": "Too many requests from this connection. Try again in a little while.", "full": "So many people have asked for updates just now that the list is not taking new addresses for a while. Please try again later.", "failed": "We could not add you just now. Try again in a moment." }, "home": { "headline": "Build a testbench. Get it scored. Climb the leaderboard.", "lede": "Download a standard IP block with its specification and RTL, build the verification collateral you would ship, and enter it. It is rated against a published rubric with every finding cited to the file and line, and your best score on each sample stands on the leaderboard.", "getStarted": "Get started", "enter": "Enter your testbench", "browseSamples": "Browse the samples", "readMethodology": "How scoring works", "updates": "Not ready to enter yet? Get updates by email.", "counters": { "samples": { "label": "samples", "definition": "IP blocks you can download and enter today." }, "dimensions": { "label": "dimensions per entry", "definition": "Every entry is rated against all six. The specification ships with the sample, so it is not rated." }, "entries": { "label": "entries", "definition": "Archives entered against a published sample, whatever the queue has since done with them." }, "entrants": { "label": "engineers", "definition": "Engineers with at least one entry. Registering is not entering." }, "ratedDimensions": { "label": "dimensions rated", "definition": "Readings the raters reported. Six per pass, each over a whole testbench." }, "tokens": { "label": "tokens processed", "definition": "Prompt and completion tokens every reported pass spent, its six raters included." }, "failed": "We could not load the counters" }, "scored": { "eyebrow": "What gets scored", "title": "Six dimensions, one rubric each", "lede_published": "Every dimension is scored out of 100 against a published rubric, criterion by criterion, with evidence cited down to file:line. The specification is not scored: it ships with the sample, so it is not your work.", "lede_unpublished": "Every dimension is scored out of 100, criterion by criterion, with evidence cited down to file:line, and your scorecard names every criterion that was applied to your entry. The specification is not scored: it ships with the sample, so it is not your work." }, "efficacy": { "chip": "The load-bearing verdict", "title": "A testbench that finds no bugs has not done its job.", "body": "Alongside the averaged score, every entry passes a bug-finding efficacy gate. A checker that cannot disagree with a wrong DUT, a corner the suite can never reach, a field the monitor cannot observe or an error class with no bin are the kind of finding that caps it, and a capped entry cannot be marked ready however high the structural scores run." }, "anyTool": { "eyebrow": "Build it your way", "title": "With ChatGPT, with Claude, or by hand.", "body": "However you produce your collateral, it is welcome. The raters read what you enter, never how it was made, and every entry is scored against the same rubric.", "ways": { "chatgpt": { "title": "With ChatGPT", "body": "Draft the plan, the agents or the assertions with ChatGPT, then review and enter them." }, "claude": { "title": "With Claude", "body": "Let Claude write some or all of it, and enter what you would stand behind." }, "byHand": { "title": "By hand", "body": "Write every line yourself, or mix hand-written work with any assistant you like." } }, "examples": "ChatGPT and Claude are examples. Any assistant, editor or workflow is welcome." }, "standings": { "eyebrow": "Overall standings", "full": "Full leaderboard", "empty": { "title": "Nobody has been scored yet", "body": "The first rated entry starts this board." } } }, "samples": { "title": "Standard samples", "about": { "title": "What a sample is", "body": "Each sample ships a micro-architecture specification, its supporting protocol documents, and the complete RTL with a filelist. Take it away, build the collateral you would ship, and enter it. Nothing you build is shown to another competitor." }, "filters": { "search": "Search samples, protocols, top modules", "difficulty": "Difficulty", "protocol": "Protocol" }, "card": { "entries_zero": "No entries yet", "entries_one": "{{count}} entry", "entries_other": "{{count}} entries" }, "unpublished": { "title": "The library opens with its first sample", "body": "Every sample is a real IP block: its micro-architecture specification, the protocol documents behind it, and the complete RTL with a filelist. Each one is curated before it is published, and none has been published yet.", "next": { "visitor": "Registering is what unlocks a sample's download, so an account made now is ready the day the first one is published.", "entrant": "Your account is ready, so a sample's download opens to you the day the first one is published. Until then, read how every entry is rated.", "operator": "Nothing reaches this page until an operator publishes it. Add a sample and upload its files from the sample library in Admin settings." }, "signUp": "Create an account", "publish": "Open the sample library", "methodology": "How rating works" }, "noMatch": { "title": "No sample matches those filters", "body": "Try another search, difficulty or protocol, or clear the filters to see the whole library.", "clear": "Clear filters" }, "failed": { "title": "We could not load the library" } }, "sample": { "filesTitle": "What ships with it", "documents": "Specifications and supporting documents", "source": "RTL source", "noFiles": "The bundle for this sample has not been recorded yet.", "gradedTitle": "How it is rated", "gradedLede": "Six raters run against your entry. Each scores its own criteria out of 100 and cites evidence down to file:line. The overall is the weighted mean of every criterion that could be rated.", "compete": "Compete on this sample", "download": "Download sample ({{size}})", "bundle": { "downloading": "Downloading", "contents": "One zip, with the specification in spec/ and the RTL in rtl/.", "notUploaded": "The collateral for this sample has not been uploaded yet.", "preparing": "The download for this sample is still being prepared. Try again in a minute.", "failed": "The download could not be prepared. Try again in a moment." }, "upload": "Enter this sample", "registerPrompt": "Everything else here reads without an account. Registering is what unlocks the download.", "viewLeaderboard": "View leaderboard", "notFound": { "title": "No such sample", "body": "That link points at a sample that is not published, or never existed.", "action": "Back to the library" }, "failed": { "title": "We could not load this sample" }, "facts": { "topModule": "Top module", "rtl": "RTL", "rtlLines": "{{count}} lines", "files": "Files", "entries": "Entries" }, "fileTable": { "kind": "Kind", "name": "Name", "size": "Size", "pagesAndSize_one": "{{count}} page, {{size}}", "pagesAndSize_other": "{{count}} pages, {{size}}" }, "fileKinds": { "spec": "MAS", "protocol": "Protocol", "registers": "Registers", "rtl": "RTL", "filelist": "Filelist", "doc": "Supporting" }, "registerToDownload": "Register to download" }, "enter": { "title": "Enter the benchmark", "help": { "title": "How entering works", "sample": "Choose a sample from the library, download its bundle from the sample's page, and enter the collateral you built for it.", "after": "It is one zip. Once it is sent you land on the entry's scorecard, which says how far the rating has got." }, "sample": { "label": "Sample", "hint": "Download its bundle from the sample's page, build your collateral, and enter it as one zip.", "openPage": "Open the sample's page", "clear": "Clear the sample", "noMatches": "No published sample matches.", "typeForMore": "Type a name to find more samples.", "unknown": "That sample is not in the library. Choose one from the list.", "unreadable": "That sample could not be loaded just now. Try again in a moment.", "noneYet": "No sample is published yet. The library opens with its first one.", "failed": "The library could not be loaded just now. Try again in a moment." }, "needSample": "Choose a sample first.", "needArchive": "Choose the archive holding your collateral first.", "tooLarge": "That archive is larger than {{size}}. Leave waveforms and build products out and try again.", "rulesTitle": "What an archive may be", "exampleLead": "For example. Any layout works: bug_report.md is the only name the raters expect.", "rules": { "format": "One .zip. The format is read off the bytes, so the extension is a courtesy rather than a check.", "size": "At most {{size}} compressed, {{unpacked}} unpacked, and {{files}} files.", "compressedSize": "At most {{size}} compressed.", "contents": "Source only. Waveforms and build products are not rated, so leave them out.", "bugReport": "Call your bug report bug_report.md. It is the one file the raters expect by name.", "images": "Every image is checked automatically on our servers before the archive is kept: at most 64 of them, each a PNG, JPEG, WebP, GIF, BMP or single-page TIFF of at most 8 MB and 16 megapixels. No sexually explicit image is accepted.", "links": "Plain files and folders. A symbolic link that points at a file or folder inside the archive is replaced by a copy of it; one that points outside is left out, and the scorecard says how many." }, "archiveLabel": "Your archive", "dropHint": "Drop one .zip anywhere on this page, or choose it below.", "drop": { "title": "Drop your file here", "hint": "One .zip, at most {{size}}.", "kept": "It will wait here while you finish. {{reason}}", "busy": "Your entry is still sending", "whileSending": "That file was not taken: an entry is already sending. Wait for it to finish before choosing another.", "tooMany_other": "That was {{count}} files, and an entry is one. Put everything into one .zip and drop that.", "folder": "That is a folder. Compress it into one .zip and drop the zip.", "nothing": "Nothing dropped could be read as a file. Choose the archive below instead." }, "step": { "sample": "Start here: choose a sample", "archive": "Add your archive", "required": "(required)" }, "chosen": "Ready to send: {{name}}, {{size}}", "nothingChosen": "No archive chosen yet.", "chooseDifferent": "Choose a different file", "removeArchive": "Remove", "advanced": "Advanced", "notes": "Notes to the raters (optional)", "notesPlaceholder": "Anything about layout or intent the raters should know, such as where the predictor lives or why a dimension is deliberately absent.", "notesHelp": "Read as context, never as instructions. Notes cannot change a score.", "progress": "Uploading your archive", "sending": "Sending your archive. Leave this page open until it finishes.", "action": "Enter this archive", "failed": "That entry could not be sent. Try again in a moment." }, "leaderboard": { "title": "Leaderboard", "pages": "Leaderboard pages", "sampleFilter": { "label": "Sample", "all": "All samples", "clear": "Show every sample", "noMatches": "No published sample matches.", "typeForMore": "Type a name to find more samples.", "everyEntry_one": "One entry ranks on the board of every sample.", "everyEntry_other": "{{count, number}} entries rank on the board of every sample: each contestant's best on each sample.", "everyEntryInWeek_one": "One entry submitted in this week ranks on the board of every sample.", "everyEntryInWeek_other": "{{count, number}} entries submitted in this week rank on the board of every sample: each contestant's best on each sample.", "sampleEntries_one": "One entry ranks on this sample's board.", "sampleEntries_other": "{{count, number}} entries rank on this sample's board: each contestant's best on it.", "sampleEntriesInWeek_one": "One entry submitted in this week ranks on this sample's board.", "sampleEntriesInWeek_other": "{{count, number}} entries submitted in this week rank on this sample's board: each contestant's best on it." }, "week": { "allTime": "Back to all time", "heading": "Week of {{from}}", "previous": "Previous week", "next": "Next week", "upcoming": "This week has not started. Nothing can have been entered in it yet.", "open": "This week is still running. Entries made before it closes still count towards it.", "settling_one": "This week has closed, and {{count}} entry in it is still being rated.", "settling_other": "This week has closed, and {{count}} entries in it are still being rated.", "final": "This week has closed and everything entered in it has been rated." }, "columns": { "rank": "Rank", "sample": "Sample", "contestant": "Contestant", "score": "Score" }, "search": { "placeholder": "Search", "tooShort_one": "Type at least {{count}} character to search.", "tooShort_other": "Type at least {{count}} characters to search.", "matches_zero": "No entry matches.", "matches_one": "{{count}} entry matches.", "matches_other": "{{count}} entries match." }, "noMatches": { "title": "Nothing matches \"{{search}}\"", "body": "No entry on this board has that in its sample, its contestant's name or an organization they show. Check the spelling, or clear the search to see every entry." }, "empty": { "title": "Nobody has been rated yet", "body": "The board fills in as entries are scored. Rating is being built; the library, the rubric and entering are live, so you can start now." }, "quietSample": { "title": "Nobody has been rated on {{sample}} yet", "body": "This board fills in as entries on this sample are scored. Download it from its page and enter, and yours could be the first row here." }, "unknownSample": { "title": "There is no sample by that name", "body": "The library holds no published sample at this address. The link may be mistyped or the sample withdrawn. Choose another sample above, or read the board for every sample." }, "quietWeek": { "title": "Nobody was rated in this week", "body": "A week ranks the entries made inside it, and nothing entered in this one has a score. Pick another week, or read the all-time board." }, "upcomingWeek": { "title": "This week has not started", "body": "A week ranks the entries made inside it, and this one has not opened yet, so there is nothing in it to rank. Step back to a week that has run, or read the all-time board." }, "pastTheEnd": { "title": "There is no page that far down", "body_one": "The standings run to {{count}} page. Every rated entry is on it.", "body_other": "The standings run to {{count}} pages. Every rated entry is on them.", "action": "Back to the first page" }, "failed": { "title": "We could not load the standings" }, "howItWorks": { "title": "How ranking works", "entries": "Every row is one entry: a sample and who entered it, ranked by that entry's score.", "bestCounts": "Only your highest-scoring entry on each sample appears. Re-entering can raise your place and never lowers it.", "ties": "Where two entries score the same, the one entered first ranks higher.", "names": "Every row shows the contestant's first and last name. Their organization appears only if they turn it on in account settings.", "weekly": "A weekly prize is decided from that week's board as it stood when the week was announced. A later appeal can raise an entry, and never takes a prize back." }, "podium": { "label": "The top three", "by": "by {{contestant}}", "unclaimed": "Unclaimed", "unclaimedBody": "Open to the next rated entry" }, "unnamed": "Unnamed contestant", "openScorecard": "Open the scorecard for {{sample}} by {{contestant}}", "scoreBar": "Score" }, "contestantPage": { "failed": "We could not load this page", "notFound": { "title": "No such contestant", "body": "Nobody competing here has a page at this address. The link may be out of date, or the account may have been closed.", "action": "See the leaderboard" }, "edit": "Edit your page", "private": { "title": "This page is private", "body": "{{name}} keeps their page private. Their name still appears beside their entries on the leaderboard.", "ownerBody": "Your page is private, so this is all anybody else sees. Turn off Private under Visibility in account settings to show your picture, bio, links, country and ranked entries." }, "bioTitle": "About", "noBio": "{{name}} has not written a bio yet.", "linksTitle": "Links", "linkLabel": "{{site}}: {{link}}", "entriesTitle": "Ranked entries", "noEntries": { "title": "No ranked entries yet", "body": "An entry {{name}} makes appears here once it is rated and ranks on the leaderboard." }, "rank": "#{{rank}}", "columns": { "rank": "Place", "sample": "Sample", "score": "Score" }, "sites": { "github": "GitHub", "gitlab": "GitLab", "bitbucket": "Bitbucket", "linkedin": "LinkedIn", "x": "X", "bluesky": "Bluesky", "mastodon": "Mastodon", "youtube": "YouTube", "scholar": "Google Scholar", "orcid": "ORCID", "researchgate": "ResearchGate", "arxiv": "arXiv", "stackoverflow": "Stack Overflow", "huggingface": "Hugging Face", "kaggle": "Kaggle", "medium": "Medium", "substack": "Substack", "website": "Website at {{host}}" }, "settings": { "view": "View your page", "private": "Private", "privateOn": "Only your name, and your organization if you show it above, appear on your page. Your picture, bio, links, country and ranked entries stay hidden.", "privateOff": "Anybody can see your picture, bio, links, country and ranked entries on your page. Your email address and phone number are never shown.", "bio": "Bio", "write": "Write", "preview": "Preview", "previewEmpty": "Nothing to preview yet.", "bioHelp": "Markdown: **bold**, *italic*, lists and links. {{used}} of {{limit}} characters.", "bioTooLong": "A bio is at most {{count}} characters.", "links": "Links", "linksHelp_one": "Up to {{count}} link, each a full https:// address. GitHub, LinkedIn, Google Scholar and other well-known sites show their own mark.", "linksHelp_other": "Up to {{count}} links, each a full https:// address. GitHub, LinkedIn, Google Scholar and other well-known sites show their own mark.", "linkLabel": "Link {{number}}", "linkInvalid": "Use a full https:// address, such as https://github.com/you.", "addLink": "Add a link", "removeLink": "Remove this link", "invalid": "Fix the fields marked in red to save.", "country": "Country", "countryHelp": "Shown with its flag on your page, and only while your page is not private." } }, "submissions": { "title": "My entries", "help": { "title": "About your entries", "body": "Everything you have entered, and how far each one has got. Nobody else can see your collateral.", "open": "Select an entry to open its scorecard, including one that is still waiting to be rated." }, "columns": { "sample": "Sample", "state": "State", "score": "Score", "submitted": "Submitted" }, "states": { "queued": "Queued", "grading": "Rating", "scored": "Scored", "failed": "Failed", "held": "Held for review", "taken_down": "Taken down", "cancelled": "Deleted" }, "empty": { "title": "Nothing entered yet", "body": "Pick a sample, build the verification collateral you would ship, and enter it as one archive. Every entry and its scorecard gathers here, and no other entrant ever sees your collateral.", "action": "Make your first entry" }, "failed": { "title": "We could not load your entries" }, "unscored": "Not rated yet", "cancel": { "action": "Delete this entry", "modal": { "title": "Delete your entry on {{sample}}?", "body": "This permanently deletes the files you uploaded and closes this entry's public scorecard. If it has a score, the score leaves the leaderboard. It cannot be undone.", "allowance": "An entry that was never rated stops counting toward your five entries a day. A rated entry still counts, and you can enter the same archive again.", "keep": "Keep it", "confirm": "Delete this entry", "failed": "That could not be deleted. Try again in a moment." } } }, "hints": { "submissionStates": { "queued": "Entered and waiting for a rater to pick it up.", "grading": "A rater is reading this entry now.", "scored": "A pass finished and its score stands.", "failed": "The last pass could not finish. That is a fault on our side, and an operator can run it again.", "held": "The pre-scan asked a person to look at this entry before it is rated. Nothing is lost while it waits.", "taken_down": "An operator took this entry down. It is off the leaderboard, and its scorecard says why.", "cancelled": "Its entrant deleted this entry. It is off the leaderboard, and its archive is deleted." }, "fileSize": "How large this file is.", "difficulty": "How demanding this block is to verify, as the library rates it.", "protocol": "A protocol this block speaks.", "fileKinds": { "spec": "The micro-architecture specification the block is built to.", "protocol": "A protocol specification the block follows.", "registers": "The block's register map.", "rtl": "The block's RTL source.", "filelist": "The list of source files, in the order they compile.", "doc": "Further material that supports the specification." }, "email": { "verified": "This address was confirmed with a one-time code.", "unverified": "This address has not been confirmed with its one-time code yet." }, "currentSession": "The session you are using right now, in this browser.", "twoFactor": { "enabled": "Signing in asks for a code from your authenticator app as well as your password.", "disabled": "Signing in asks only for your password.", "unknown": "We could not read whether two-factor sign-in is on." }, "roles": { "operator": "Runs this deployment: curates the sample library and reads the roster.", "contestant": "Enters the benchmark, and holds no operator powers." }, "entrantStatuses": { "active": "In good standing: may enter and appeal.", "suspended": "Suspended: cannot enter or appeal, and their entries are off the public leaderboard.", "suspendedBecause": "Suspended: cannot enter or appeal, and their entries are off the public leaderboard. The reason given: {{reason}}", "unverified": "Has not confirmed their email address yet." }, "sampleStates": { "published": "In the public library, where anybody can read it.", "draft": "Not published, so only operators can see it." }, "listedOnly": "This file is listed but its bytes never arrived, so the download leaves it out.", "setupStep": { "done": "This step is complete.", "todo": "This step still needs doing before rating can go live." }, "budget": { "rating": "Within the daily token budget, so the queue is starting passes.", "approaching": "Most of the daily token budget is spent. Passes still start until it runs out.", "paused": "The daily token budget is spent, so no new pass starts until the window rolls on." }, "githubToken": { "expired": "This token has expired, so nothing is filed until a new one is saved.", "expiresSoon": "This token expires soon. Save a new one before it does." }, "slackWebhook": { "posting": "Slack accepted this webhook the last time it was asked, and notices are being posted through it.", "revoked": "Slack answered that this webhook is gone, so nothing is posted until a new one is saved here, or a test message posts once the channel or the app is back.", "expired": "The date you chose to rotate this webhook by has passed. It still posts; save a new one to keep to the schedule.", "expiresSoon": "The date you chose to rotate this webhook by is close. Save a new one before it." }, "skills": { "inUse": "The version new passes run.", "notUploaded": "Its archive never arrived, so it cannot be used to rate.", "preflight": { "pending": "Nobody has read this version's rubrics yet. The first pass under it reads them, once, before anything is rated.", "succeeded_one": "Its rubrics were read {{when}}: {{count}} criterion and its weight.", "succeeded_other": "Its rubrics were read {{when}}: {{count}} criteria and their weights.", "failed": "Reading its rubrics failed, so nothing is rated under this version until an operator runs it again. {{reason}}" } }, "reportOrigin": "Which part of the deployment hit this fault.", "reportStates": { "queued": "Waiting to be filed as a GitHub issue.", "open": "Filed as a GitHub issue.", "failed": "Filing it as a GitHub issue failed. The next occurrence tries again.", "withheld": "Held back by the hourly limit on new issues, and filed once it allows.", "unconfigured": "Counted, but not filed, because no GitHub token is saved." }, "llmAccount": { "pinned": "Every new rating pass signs in with this account. Choose another under rating setup before turning it off." }, "credentialExpiry": { "expired": "The expiry date this account was given, {{date}}, has passed, so it cannot sign in.", "soon": "The expiry date this account was given, {{date}}, is close. Rotate it before then." }, "credentialProvider": "The kind of account this is, which is fixed once it is registered.", "efficacyGate": "The gate every entry passes beside its averaged score: a capped efficacy criterion blocks a ready to use sign-off.", "filterCount": "How many accounts this filter holds.", "invitedRoles": { "operator": "Will run this deployment from the moment they accept the invitation.", "contestant": "Will enter the benchmark, and holds no operator powers." }, "invitationsWaiting": "How many invitations nobody has accepted yet, expired ones included.", "invitationExpired_one": "The link stopped working after {{count}} hour. Resend the invitation to send a fresh one.", "invitationExpired_other": "The link stopped working after {{count}} hours. Resend the invitation to send a fresh one.", "moderationStatuses": { "open": "Nobody has reviewed this report yet.", "dismissed": "An operator read this report and left what it reported as it was.", "actioned": "An operator took down what this report was about.", "rescanned": "An operator sent the entry back to the pre-scan, which ran it again.", "withNote": "{{status}} Their note: {{note}}" }, "moderationSources": { "user": "A person reported this, in the words beside it.", "prescan": "The automated pre-scan flagged this entry before it was rated, or could not judge it, and it is held until somebody decides.", "wordlist": "The abusive-words list matched what a person wrote in a comment." }, "reportTabCount": "Reports on this tab nobody has reviewed yet.", "alertStatuses": { "on": "This admin is emailed about every kind they hear about.", "off": "This admin is kept, and nothing is emailed to them until they are turned back on." }, "feedbackStandings": { "awaiting": "The entrant wrote last and nobody has answered yet.", "open": "This conversation takes messages from both sides.", "closed": "An operator closed this conversation, so it takes no more messages until it is reopened." }, "feedbackAwaitingCount": "Open conversations in which the entrant wrote last and nobody has answered yet.", "awaiting": { "entries": "Entries whose rating failed and that wait on an operator to run them again. Entries the pre-scan held are counted under Reported content.", "reports": "Reports nobody has reviewed yet, across entries, scorecards, comments and contestants' pages.", "feedback": "Open conversations in which the entrant wrote last and nobody has answered yet.", "llms": "LLM accounts that need a person: refused by the vendor, held with no end, or lapsed or lapsing within the warning window.", "github": "The error reporting token has lapsed or lapses within the warning window, and waits on a new one.", "jira": "The Jira token has lapsed or lapses within the warning window, or Jira refused the connection the last time it was used." }, "jiraToken": { "expired": "This token has expired, so known bugs cannot be read and nothing is filed until a new one is saved.", "expiresSoon": "This token expires soon. Save a new one before it does." }, "feedbackByEmail": "Sent by replying to our email. The email it quoted, any signature and any attachment were left out.", "feedbackReplies": { "on": "Replies to a reply email land in their conversation, from its author only." }, "refusalKinds": { "not_pinned": "Refused because no rater was pinned, so nothing could rate it. Ours to fix.", "unscreened": "Refused because it held an image and the explicit-content check could not run. Ours to fix.", "account": "Refused because the account could not enter: the current terms not accepted, no name given, or a suspension.", "sample": "Refused because no published sample goes by the name it was entered against.", "daily_limit": "Refused because the account had already made its five entries in the last day.", "archive": "Refused because the archive, or the form it came in, broke a rule: not a zip, too large, damaged, or missing a part.", "image": "Refused because an image in it was judged sexually explicit or could not be read.", "duplicate": "Refused because an identical archive had already been entered." }, "emailDelivery": { "on": "Brevo tells this site whenever it stops delivering to an address." }, "undeliverable": { "unsubscribed": "Its owner unsubscribed, usually with the button their mail client draws on our emails.", "hard_bounce": "The address does not exist, or its mail server refuses our email for good.", "spam": "Its owner marked one of our emails as spam.", "blocked": "Brevo declined to send to it because the address is on its blocklist." } }, "scorecard": { "backToSample": "About {{sample}}", "percent": "{{percent}}%", "noScore": "Not rated", "nothingRated": "This pass found nothing it could rate, which is a finding rather than a score.", "signOff": "Sign-off: {{signOff}}", "efficacyTag": "efficacy", "cappedTag": "capped", "origins": { "initial": "First pass", "appeal": "Re-grade on appeal", "operator_rerun": "Re-run by an operator" }, "efficacy": { "title": "Bug-finding efficacy", "ready_to_use": { "label": "Ready to use", "body": "Every efficacy criterion that applies to this block was scored clear of its cap." }, "blocked": { "label": "Blocked", "body": "At least one efficacy criterion sits at or below its cap, so this entry cannot be marked ready to use whatever the percentage says. A testbench that finds no bugs has not done its job." }, "unproven": { "label": "Unproven", "body": "Nothing was capped, and nothing was shown either. Some efficacy criteria need a simulator, and the raters read but never run, so this is a gap in the evidence rather than a finding against your work." }, "cappedBy": "What blocked the sign-off", "unsettled": "Waiting on a simulator" }, "sections": { "title": "Section scores", "lede": "A dimension's score is the weighted mean over the criteria that were scored, reported out of 100. Each criterion is judged on the rubric's four bands and shown out of 100 too. A criterion that does not apply keeps its rubric weight and drops out of the average entirely, so it is never a zero." }, "dimension": { "notRated": "Not rated", "whyNotRated": "Why it was not rated:", "findings": "What the rater found", "showReport": "Show the full report", "hideReport": "Hide the full report", "recommendation": "Recommendation: {{recommendation}}" }, "columns": { "code": "ID", "criterion": "Criterion", "weight": "Weight", "score": "Mark", "evidence": "Evidence" }, "verdicts": { "scored": "Scored", "not_applicable": "Not applicable", "pending": "Pending" }, "missing": { "title": "What is missing", "lede": "Ranked by risk. Efficacy findings come first, because each one names a class of bug this environment would not catch.", "empty": "Nothing was scored at or below the middle of the band, so there is no gap to name here.", "issue": "Issue", "recommendation": "Recommendation" }, "failure": { "attempts": "Attempt {{attempts}} of {{max}}", "nextAttempt": "The next pass runs {{when}}.", "retrying": { "title": "A pass did not finish", "body": "Something at our end stopped the rating partway through, and the queue will run it again on its own. Nothing about your collateral caused this, and there is nothing for you to do." }, "given_up": { "title": "We could not rate this entry", "body": "Every pass this entry was allowed went out against a fault at our end, so the queue has stopped trying. An operator will run it again, which costs you nothing, and an entry we never managed to rate does not count against the entries you may make today." }, "archive": { "title": "Your archive could not be unpacked", "body": "The rating pass could not unpack the archive you entered, so this entry has no score and the queue will not try again: the same archive gives the same answer every time. It does not count against the entries you may make today. Fix the archive and enter it again." }, "scored": { "retrying": { "title": "A re-grade did not finish", "body": "The score below still stands. The re-grade stopped partway through for a reason at our end, and the queue will run it again on its own. A re-grade is only ever kept when it beats what you already have, so nothing here can lower your score." }, "given_up": { "title": "The re-grade could not finish", "body": "The score below still stands, and nothing that happened here can lower it. The re-grade went out against a fault at our end and the queue has stopped trying, so an operator will run it again, which costs you nothing." }, "archive": { "title": "The re-grade could not unpack your archive", "body": "The score below still stands, and nothing that happened here can lower it. The re-grade could not unpack the archive this entry holds, and the queue will not try again, because the same archive gives the same answer every time." } }, "rules": { "lede": "What the pass found:", "contradiction": "two of its members cannot both be written to disk, such as a file and a folder with the same name, or a name the filesystem will not take.", "file_and_folder": "it lists a file and a folder under the same name, which no disk can hold.", "header_disagrees": "a member's header does not name what the archive's directory lists there.", "too_many_entries": "unpacking it, with every symbolic link resolved to a copy, makes more files than an entry may hold.", "bomb": "unpacking it, with every symbolic link resolved to a copy, takes more space than an entry may hold.", "other": "it broke one of the rules on the Enter page." } }, "notRated": { "queued": { "title": "Waiting for a rater", "body": "This entry is in the queue. Rating takes minutes and runs against the model and skills version this deployment is pinned to, and this page fills in as soon as the pass finishes." }, "queue": { "behind": "This entry is in the queue, and the queue is behind at our end; your entry is safe and nothing is charged. This page fills in as soon as the pass finishes.", "paused": "This entry is in the queue, and rating is paused at our end for the day's budget; your entry is safe, nothing is charged, and it resumes on its own. This page fills in as soon as the pass finishes." }, "grading": { "title": "Being rated now", "body": "Six raters are reading your collateral. Come back shortly: the scorecard appears here when the pass finishes." }, "failed": { "title": "That pass could not finish", "body": "The rating pass failed for a reason at our end rather than anything in your collateral, and an operator can run it again. Any score you had already earned still stands." }, "scored": { "title": "The raters could not finish", "body": "This entry reads as scored because a pass ran on it, and that pass could not finish. That is a fault at our end rather than anything in your collateral: an operator can run it again, a re-run costs you nothing, and the one appeal you are entitled to stays yours." }, "held": { "title": "Not rated yet", "body": "Nothing rates this entry until the review is finished. Once it is cleared it is rated as usual." }, "taken_down": { "title": "Not rated", "body": "This entry was taken down before anything rated it, so nothing will." }, "cancelled": { "title": "This entry was deleted", "body": "Its entrant deleted it, so nothing rates it again. It is off the leaderboard and its archive is deleted." } }, "appeal": { "title": "Disagree with this score?", "body": "A score comes from one pass against the model and skills version recorded above. If you believe a rater misread your entry, ask for one re-grade. The higher of the two scores stands, so an appeal can never lower your rank.", "start": "Request a re-grade", "label": "What did the raters get wrong?", "placeholder": "rate-sva returned N/A, but my assertions are under testbench/verification_ip/.", "help": "At least {{min}} characters, and one re-grade per entry.", "send": "Request re-grade", "failed": "We could not lodge that appeal. Try again in a moment.", "spent": { "title": "Appeal lodged", "body": "This entry will be re-graded once, against the same pinned model and skills version. The higher of the two scores stands, so the appeal can never lower your rank.", "argumentTitle": "What you told us" }, "unavailable": { "title": "Nothing to appeal yet", "body": "An appeal is a disagreement with a rating, so it opens once this entry has one. A pass that could not finish is ours to re-run, and it never costs you the re-grade you are entitled to." } }, "failed": { "title": "We could not load this scorecard" }, "notFound": { "title": "No such entry", "body": "No scorecard you can read lives at that address. It may have been removed, its contestant may be off the leaderboard, or it may belong to somebody else and not be ranked yet: a contestant's own entries open only to them until the leaderboard ranks one.", "action": "Back to the leaderboard" }, "finding": { "affects": "Affects {{codes}}" }, "bands": { "Poor": "Poor", "Partial": "Partial", "Good": "Good", "Excellent": "Excellent" }, "counts": { "scored": "{{count}} scored", "notApplicable": "{{count}} not applicable", "pending": "{{count}} pending", "waiting": "{{count}} waiting on a simulator", "none": "No criteria" }, "hints": { "rankOverall": "Where this entry stands among every ranked entry. Only each contestant's best entry on each sample is ranked.", "rankSample": "Where this entry stands among the ranked entries on this sample. Opens that sample's leaderboard.", "unranked": "This entry has no score on the leaderboard, so it holds no place there yet.", "appeal": { "available": "You may ask for one re-grade of this entry, and the higher of the two scores stands. Select to open it.", "lodged": "You have asked for your one re-grade. Select to read where it stands.", "unavailable": "There is no score to appeal yet. Select to read more." }, "vendor": "The company whose model read this entry.", "model": "The model the raters ran on. Two scores are comparable only when the same model and skills version produced them.", "skillsVersion": "The version of the rating skills that marked this entry. Together with the model, it is what the deployment rates under, so scores read under the same model and skills version were marked the same way.", "origins": { "initial": "The first pass over this entry after it was entered.", "appeal": "The re-grade the contestant asked for. The higher of the two scores stands.", "operator_rerun": "A pass an operator started again after a fault on our side. It costs the contestant nothing." }, "bands": { "Poor": "Below {{partial}} out of 100: the collateral misses most of what the rubric asks for.", "Partial": "From {{partial}} to just under {{good}} out of 100: some of what the rubric asks for is there, with real gaps.", "Good": "From {{good}} to just under {{excellent}} out of 100: most of what the rubric asks for is there.", "Excellent": "{{excellent}} out of 100 or higher: close to everything the rubric asks for is there." }, "unknownBand": "The band this score falls in, as the rubric names it.", "critical": "The rater found a critical defect in this dimension.", "efficacyTag": "One of the bug-finding efficacy criteria, which decide whether the entry can be marked ready to use.", "cappedTag": "A critical defect holds this criterion at or below its cap, whatever else it showed.", "verdicts": { "scored": "The rater judged this criterion against its four anchors, which read 0, 33.3, 66.7 and 100.", "not_applicable": "This criterion does not apply to this block, so it drops out of the average rather than counting as zero.", "pending": "Checking this needs a simulator, and the raters read but never run, so it drops out of the average rather than counting against you." }, "severities": { "0": "The most severe kind of finding: a defect that would let real bugs through.", "1": "A serious gap the rater expects to matter.", "2": "A gap worth fixing.", "3": "A minor issue.", "4": "The rater gave this finding no severity this site recognises." }, "stands": "This is the pass whose score counts. When an entry has several, the highest one stands.", "root": "A top-level folder the archive held.", "attempts": "How many times the queue has tried to rate this entry, out of the most it tries before an operator steps in.", "attemptsArchive": "The attempts this entry was charged for. The queue stopped after this one, because the same archive gives the same answer every time.", "reactions": { "choose": "{{name}}. Press to leave yours. It replaces any reaction you left before, and it is public with your name.", "yours": "{{name}} is your reaction to this scorecard. Press it again to take it back.", "readOnly": "{{name}}: how many people reacted this way. Reactions are public, with the name of whoever left one." }, "commentHidden": { "disliked_by_you": "You disliked this comment, so it stays hidden from you. Take the dislike back to see it in the list again.", "hidden": "Hidden from everybody, because enough people disliked it or it matched our list of abusive words." }, "commentMine": "You wrote this. Your own comments always come first for you, and you always see them, even when they are hidden from everybody else.", "pendingReview": "A new bug you reported that we confirmed in the RTL. A person at MooresLab reviews every one, and it earns credit once they approve it.", "confirmedNewBug": "A new bug you reported that a person at MooresLab reviewed and approved. It counts in your favour, and later entries are scored against it." }, "header": { "entered": "Entered {{date}}", "enteredExactly": "Entered on {{entered}}.", "enteredAndRated": "Entered on {{entered}}, rated on {{rated}}.", "by": "By {{name}}", "byWithOrganization": "By {{name}} at {{organization}}", "unnamed": "a contestant", "rankOverall": "#{{place}} of {{total}} on the leaderboard", "rankSample": "#{{place}} of {{total}} on this sample", "unranked": "Not on the leaderboard", "overall": "Overall score", "gate": "Efficacy: {{verdict}}", "appeal": { "available": "Re-grade available", "lodged": "Appeal lodged", "unavailable": "Nothing to appeal yet" }, "publicView": "This is the public scorecard. The evidence, the findings and the raters' reports quote the contestant's own collateral, so they stay with the contestant.", "operatorView": "You are reading this entry as an operator, so you see everything the contestant does." }, "share": { "action": "Share", "label": "Share this scorecard", "linkedin": "Share on LinkedIn", "copy": "Copy link", "copied": "Link copied", "copyFailed": "Could not copy the link", "hint": "Share this scorecard's public page. The link unfurls as a card with the sample, the name and the score, and nothing a rater wrote.", "unavailable": { "unrated": "There is nothing to share yet. Once this entry is rated and on the leaderboard, its scorecard is public and can be shared.", "offBoard": "Only a scorecard the leaderboard ranks is public, and this entry is not on the leaderboard, so a link to it would show nobody else anything." } }, "tabs": { "label": "Scorecard sections", "overview": "Overview", "dimensions": "Dimensions", "findings": "Findings", "files": "Files", "details": "Details", "comments": "Comments" }, "overview": { "dimensionsTitle": "Dimensions", "openDimension": "Open {{dimension}}", "critical": "Critical", "tallyTitle": "How the criteria scored", "tallyLede_one": "{{count}} criterion across the dimensions rated.", "tallyLede_other": "{{count}} criteria across the dimensions rated.", "capped_one": "{{count}} scored criterion is held at its cap.", "capped_other": "{{count}} scored criteria are held at their caps.", "worstTitle": "Most severe findings", "allFindings": "See every finding" }, "dimensions": { "expandAll": "Expand all", "collapseAll": "Collapse all" }, "findings": { "title": "What the raters found", "lede": "Every finding the six raters reported, most severe first. Narrow them by severity or by dimension.", "severityFilter": "Severity", "dimensionFilter": "Dimension", "any": "All", "severities": { "0": "Critical", "1": "High", "2": "Medium", "3": "Low", "4": "Unranked" }, "none": { "title": "No findings reported", "body": "The raters reported no findings for this entry. The criteria that marked lowest are still listed below." }, "filteredEmpty": "No finding matches these filters." }, "files": { "lede": "The files this entry was made of, exactly as they were entered. Only you and the people who operate this benchmark can open them." }, "details": { "arrived": { "title": "What arrived", "lede": "What the archive unpacked to when the raters read it. This is how a rater that found nothing is told apart from an archive that held nothing.", "files": "Files, copies of resolved links included", "bytes": "Unpacked size", "bugReport": "bug_report.md", "noBugReport": "The archive held no bug_report.md. That is a finding, not a reason the pass stopped.", "roots": "Top-level folders and files", "noRoots": "The archive was empty.", "links": "Symbolic links", "linksResolved_one": "{{count}} symbolic link pointed at a file or folder inside the archive and was replaced by a copy of it.", "linksResolved_other": "{{count}} symbolic links pointed at files or folders inside the archive and were replaced by copies of them.", "linksDropped_one": "{{count}} symbolic link pointed outside the archive, at nothing, or at itself, and was left out. Anything that reaches a file through it did not resolve for the raters.", "linksDropped_other": "{{count}} symbolic links pointed outside the archive, at nothing, or at themselves, and were left out. Anything that reaches a file through them did not resolve for the raters.", "stripped": "Removed before reading", "strippedBookkeeping_one": "{{count}} file an operating system wrote beside the collateral (__MACOSX, .DS_Store, Thumbs.db and the like) was removed.", "strippedBookkeeping_other": "{{count}} files an operating system wrote beside the collateral (__MACOSX, .DS_Store, Thumbs.db and the like) were removed.", "strippedVersionControl_one": "{{count}} version-control file (.git, .svn or .hg history) was removed. The raters read the collateral, not its history.", "strippedVersionControl_other": "{{count}} version-control files (.git, .svn or .hg history) were removed. The raters read the collateral, not its history.", "strippedNestedArchives_one": "{{count}} archive inside the archive was removed unopened. Nothing packed inside it was read by the raters.", "strippedNestedArchives_other": "{{count}} archives inside the archive were removed unopened. Nothing packed inside them was read by the raters." }, "spend": { "title": "What the pass spent", "total": "Tokens in total", "input": "Input", "output": "Output", "subagentInput": "Sub-agent input", "subagentOutput": "Sub-agent output", "note": "Counted in tokens rather than money. The six raters run as sub-agents of one pass, so their halves are most of the spend." }, "pin": { "title": "What rated it", "vendor": "Vendor", "model": "Model", "skills": "Skills version", "origin": "Pass", "started": "Started", "completed": "Finished", "duration": "Took", "minutes_one": "{{count}} minute", "minutes_other": "{{count}} minutes" }, "notes": { "title": "Notes to the raters", "empty": "No notes were sent with this entry." }, "passes": { "title": "Every pass", "pass": "Pass", "origin": "Why it ran", "score": "Score", "pin": "Model and skills", "tokens": "Tokens", "completed": "Finished", "stands": "Stands" }, "appeal": { "title": "The contestant's appeal", "lodged": "Lodged {{date}}. What they argued:", "none": "The contestant has not appealed this entry." } }, "comments": { "reactions": { "label": "Reactions to this scorecard", "counted": "{{name}}, {{count}}", "names": { "thumbs_up": "Thumbs up", "heart": "Heart", "shocked": "Shocked", "trophy": "Trophy", "fire": "Fire" } }, "composer": { "label": "Add a comment", "replyLabel": "Write a reply", "length": "{{count}} of {{max}} characters", "reactionLabel": "React to the scorecard too (optional)", "post": "Post", "reply": "Reply", "empty": "Write something to post.", "failed": "Your comment could not be posted. Try again in a moment." }, "gate": { "signedOut": "Sign in to comment, react, or vote on comments.", "signIn": "Sign in", "suspended": "This account is suspended, so it cannot comment or react. Your profile says why.", "unverified": "Verify your email address to comment. The banner at the top of the page sends the code again.", "unnamed": "Add your first and last name to comment. They sign everything you write here.", "terms": "Accept the current terms of service and privacy policy to comment.", "fix": { "suspended": "Open your profile", "unnamed": "Add your name", "terms": "Read the terms" } }, "removed": "Comment removed", "unnamed": "A contestant", "mine": "Yours", "byLabel": "Comment by {{name}}", "authorReacted": "Their reaction to this scorecard: {{name}}", "like": "Like", "likeCount_one": "Like. {{count}} person likes this.", "likeCount_other": "Like. {{count}} people like this.", "dislike": "Dislike. Nobody else sees it, and it hides the comment from you.", "cannotWrite": "Your account cannot vote yet. The note above the comments says why.", "signInToVote": "Sign in to vote on comments.", "reply": "Reply", "confirmRemoval": "Delete this comment?", "actionFailed": "That did not go through. Try again in a moment.", "hiddenBadges": { "disliked_by_you": "Disliked by you", "hidden": "Hidden" }, "viewReplies_one": "View {{count}} reply", "viewReplies_other": "View {{count}} replies", "hideReplies": "Hide replies", "moreReplies": "Show more replies", "more": "Show more comments", "seeHidden_one": "See hidden comments ({{count}})", "seeHidden_other": "See hidden comments ({{count}})", "seeHiddenReplies_one": "See hidden replies ({{count}})", "seeHiddenReplies_other": "See hidden replies ({{count}})", "hideHidden": "Hide them again", "hiddenFailed": "The hidden comments could not be loaded. Try again in a moment.", "repliesFailed": "The replies could not be loaded. Try again in a moment.", "failed": "We could not load the comments", "empty": { "title": "No comments yet", "body": "Be the first to say what you make of this entry." }, "allHidden": { "title": "Every comment here is hidden", "body": "They are below, behind See hidden comments." } }, "moderation": { "held": { "title": "Held for review", "body": "Before an entry is rated, an automated check reads a short summary of it. It asked a person to look at this one first. Nothing is lost while it waits, and once it is cleared it is rated as usual." }, "takenDown": { "title": "This entry was taken down", "body": "On {{date}}, an operator removed it from the leaderboard and closed its public scorecard.", "reason": "Their reason: {{reason}}" } }, "knownBugs": { "title": "Known bugs", "headline_one": "{{found}} of {{count}} known bug found", "headline_other": "{{found}} of {{count}} known bugs found", "unavailable": "Bug scoring is unavailable for this entry", "noReport": "0 of {{total}} found: no bug_report.md in this entry", "noneKnown": "No known bugs are recorded for this sample yet", "confirmedNew_one": "{{count}} new bug confirmed", "confirmedNew_other": "{{count}} new bugs confirmed", "asOf": "Counted against the known bugs as of {{date}}.", "wallLegend": "Each trophy is a known bug this entry found. Each ? is one it has not: what it is stays hidden.", "lede": "We already know of some RTL bugs in every sample. An entry's bug_report.md is checked against the RTL and the specification, then matched to them. Only RTL bugs count, and none of this changes any other score on this page.", "lines": "Lines {{start}} to {{end}} of bug_report.md", "matched": { "title": "Known bugs you found", "yours": "What you reported", "ours": "Our record", "specReference": "What the specification says", "rtlFix": "The fix in the RTL", "notRecorded": "Not recorded" }, "confirmed": { "title": "New bugs you found", "badge": "Confirmed new bug" }, "pending": { "title": "New bugs waiting on review", "badge": "Pending review" }, "other": { "title": "Claims that earned nothing", "not_a_bug": "Not a bug: the RTL does what the specification asks here.", "not_rtl_bug": "Not an RTL bug: spec and documentation issues are not scored.", "duplicate_of_claim": "Same bug as an earlier claim in your report, so it counts once.", "insufficient_detail": "Not enough detail to check: name the signal, module or behaviour that goes wrong." }, "operator": { "title": "Operator detail", "failureReason": "Why it failed", "scoringSkillsVersion": "Skills version", "noClaims": "The report made no claims.", "claimTitle": "{{id}}: {{verdict}}, lines {{start}} to {{end}}", "none": "None", "yes": "Yes", "no": "No", "fields": { "staticCheck": "Static check", "matchedKey": "Matched key", "unconfirmedKey": "Refused match", "duplicateOf": "Duplicate of", "alreadyPending": "Already pending", "shouldSplit": "Should be split", "injectionSuspected": "Injection suspected", "filedKey": "Filed as", "reviewState": "Review", "filingError": "Filing error", "reasoning": "Reasoning", "jiraDraft": "Jira draft" } } } }, "profile": { "showOrganization": "Show my organization publicly", "showOrganizationOn": "Your organization appears under your name on the leaderboard.", "showOrganizationOff": "Your organization stays private.", "organization": "Organization", "phone": "Mobile number", "phoneNote": "Only used if we need to reach you about an entry. Never verified, never shown.", "saveFailed": "That did not save. Try again.", "unavailable": "Your profile could not be loaded. Reload the page.", "removals": { "title": "An operator removed part of your page", "lede": "Whoever operates this benchmark decided it broke the terms of service, for the reason below. You can put something new in its place at any time, and the notice goes while something is there.", "item": "{{part}}, on {{date}}: {{reason}}", "parts": { "bio": "Your bio", "links": "Your links", "picture": "Your picture" } } }, "setup": { "title": "Tell us your name", "subtitle": "This is how you appear on the leaderboard.", "firstName": "First name", "lastName": "Last name", "nameRequired": "Enter your first and last name to continue.", "organization": "Organization (optional)", "organizationNote": "Hidden unless you show it in account settings.", "phone": "Mobile number (optional)", "phoneNote": "Only to reach you about an entry. Never shown.", "action": "Continue", "signOut": "Not you? Sign out", "picture": { "add": "Add a profile picture", "change": "Change your picture", "note": "Optional. Shown on your page, which stays private until you open it in account settings." } }, "verifyBanner": { "title": "Verify your email address", "body": "We sent a link to {{email}}. Open it to confirm the address is yours.", "sent": "A new link is on its way to {{email}}.", "resend": "Resend the link", "resendIn": "Resend in {{time}}" }, "suspendedBanner": { "title": "Your account is suspended", "reason": "The reason recorded: {{reason}}", "noReason": "No reason was recorded.", "means": "While it is suspended you cannot enter or appeal, your entries are off the public leaderboard, and every comment, vote and reaction you left has been deleted. You can still sign in and read your own entries." }, "operator": { "samples": { "title": "Sample library", "new": "New sample", "published": "Published", "draft": "Draft", "publish": "Publish", "unpublish": "Unpublish", "columns": { "name": "Sample", "difficulty": "Difficulty", "state": "State", "entries": "Entries" }, "search": "Search the library", "emptySearch": { "title": "No sample matches that search", "body": "The search covers drafts as well as published samples. Clear it to see the whole library." }, "empty": { "title": "The library is empty", "body": "Create a sample to get started. It is born a draft, so nothing appears publicly until you publish it." }, "failed": { "title": "We could not load the library" } }, "sample": { "notFound": "That sample does not exist.", "failed": { "title": "We could not load this sample" }, "viewPublic": "View public page", "files": "Files", "filesNote": "What a contestant downloads. A size is what the bucket counted, and removing a row removes the file behind it.", "noFiles": "No files uploaded yet.", "addFile": "Upload a file", "pages": "Pages", "dangerZone": "Danger zone", "delete": "Delete this sample", "deleteNote": "Refused once the sample has taken an entry. Unpublish it instead, which hides it from the library and leaves the results intact.", "deleteRefused": "This sample has entries against it and cannot be deleted. Unpublish it instead.", "chooseFile": "Choose a file", "chosen": "Ready to upload: {{name}}, {{size}}", "nothingChosen": "No file chosen yet. Up to {{size}} each.", "tooLarge": "That file is larger than {{size}}. Split it, or compress it, and try again.", "upload": "Upload", "uploadFailed": "That file was not uploaded. Try again in a moment.", "listedOnly": "Listed, not uploaded" }, "sampleForm": { "newTitle": "New sample", "editTitle": "Edit sample", "name": "Name", "slug": "Slug", "slugNote": "Lowercase letters, digits and single hyphens. It is in every link to this sample.", "tagline": "Tagline", "description": "Description", "protocols": "Protocols", "protocolsNote": "Comma separated.", "difficulty": "Difficulty", "topModule": "Top module", "rtlLines": "RTL lines", "jiraIp": "Jira IP", "jiraIpNote": "The option of Jira's IP field this sample's known bugs carry, such as dma_controller. Leave it blank when that is the slug.", "createDraft": "Create draft", "required": "A name and a slug are required.", "saveFailed": "That did not save. The slug may already be taken." }, "entrants": { "title": "Users", "lede": "Everybody registered. Narrow by role and standing to see who is who, search by address, name or organization, and open anybody to see their entries and their history. This is the only screen that shows an email address.", "notGiven": "Not given", "noName": "No name yet", "columns": { "person": "Person", "role": "Role", "status": "Standing", "organization": "Organization", "entries": "Entries", "joined": "Joined" }, "roles": { "operator": "Operator", "contestant": "Contestant" }, "statuses": { "active": "Active", "suspended": "Suspended", "unverified": "Unverified email" }, "filters": { "role": "Role", "status": "Standing", "everyone": "Everyone", "operators": "Operators", "contestants": "Contestants", "anyStatus": "Any" }, "search": { "placeholder": "Search people", "tooShort_one": "Type at least {{count}} character to search.", "tooShort_other": "Type at least {{count}} characters to search." }, "menu": { "label": "Actions for {{name}}", "profile": "Open profile", "grantOperator": "Make operator", "revokeOperator": "Remove operator", "suspend": "Suspend", "reinstate": "Lift suspension", "temporaryPassword": "Reset to a temporary password", "removePageContent": "Remove page content" }, "pages": "Pages of users", "empty": { "title": "Nobody has registered yet", "body": "The list fills in as engineers sign up." }, "filteredEmpty": { "title": "Nobody matches these filters", "body": "Choose another role or standing, or clear the search, to see more people." }, "failed": { "title": "We could not load the users" }, "views": { "label": "People or pending invitations", "people": "People", "invitations": "Pending invitations" } }, "entries": { "title": "Entries", "lede": "Every entry anybody has made, what state it is in, and what rating it cost. Filter by state to find the ones that are stuck.", "filter": "State", "anyState": "Every state", "noName": "No name yet", "notRated": "Not rated", "tokens_one": "{{tokens}} tokens in 1 pass", "tokens_other": "{{tokens}} tokens in {{count}} passes", "costNote": "Every pass this entry took, sub-agent tokens included.", "queue": { "waiting": "Waiting", "leased": "Being rated", "delayed": "Delayed", "oldest": "Longest wait", "nothingWaiting": "Nothing waiting", "spend": "Day's spend", "spendValue": "{{spent}} of {{ceiling}}", "paused": "Rating is paused: the day has spent every token it is allowed. Entries are still accepted and are queueing up, unspent, and nothing here has failed. It starts again on its own as the window rolls, or as soon as the ceiling on the usage page is raised." }, "columns": { "entrant": "Contestant", "sample": "Sample", "state": "State", "score": "Score", "cost": "Cost", "submitted": "Entered" }, "rerun": "Run again", "empty": { "title": "Nobody has entered yet", "body": "Entries appear here as engineers send them, and each row fills in as it is rated." }, "emptyFiltered": { "title": "No entry is in that state", "body": "Choose another state, or every state, to see the rest." }, "failed": { "title": "We could not load the entries" }, "menu": { "label": "Actions for {{name}}'s entry on {{sample}}", "labelUnnamed": "Actions for this entry on {{sample}}", "remove": "Remove entry" }, "removeFromScorecard": "Remove entry" }, "rerun": { "title": "Run {{sample}} again", "lede": "A re-run is ours, not the contestant's appeal. It queues a fresh pass, it spends none of their one re-grade, and every pass is kept with the highest score standing, so it can raise a score and never lower one.", "reason": "Why this needs another pass", "required": "Say why. A pass costs millions of tokens, and the log has to answer what it was for.", "confirm": "Queue the pass", "failed": "That could not be queued. Try again in a moment." }, "suspend": { "title": "Suspend {{name}}", "lede": "A suspended account cannot enter or appeal, and its rows leave the leaderboard. Lifting the suspension restores both.", "purge": "Every comment, like, dislike and reaction this account has left is deleted permanently. Lifting the suspension later does not bring them back.", "reason": "Reason", "required": "Say why. The contestant is shown this.", "confirm": "Suspend account", "failed": "That did not save. The account may no longer exist." }, "llms": { "title": "LLMs", "lede": "The LLM accounts rating passes sign in with: a Claude subscription or an API key each. A secret goes in once and never comes back out. Which account rates entries is chosen under rating setup.", "table": "LLM accounts", "pinned": "Rating with this", "pinUnknown": "We could not tell which account rates entries, so none is marked below." }, "raters": { "title": "Rating setup", "lede": "What rates entries. Work through the steps in order; the banner says when rating is live. What rating may spend is on the Usage page.", "neverUsed": "Never used", "noError": "None", "enable": "Enable", "disable": "Disable", "statusRefused": "That did not change. The account may no longer be registered.", "providers": { "claudeSubscription": "Claude subscription", "anthropicApiKey": "Anthropic API key", "openaiApiKey": "OpenAI API key", "geminiApiKey": "Gemini API key" }, "statuses": { "active": "Active", "exhausted": "At its limit", "invalid": "Rejected", "disabled": "Off", "expired": "Expired" }, "notes": { "active": "Ready to rate.", "exhausted": "Waiting on a usage window. It comes back on its own.", "exhaustedUntil": "Comes back on its own {{when}}.", "invalid": "The vendor refused this secret. Paste a new one in.", "disabled": "Turned off here. Nothing signs in with it.", "expired": "Past the expiry recorded for it. Paste in a new secret with its new date." }, "columns": { "label": "Account", "provider": "How it signs in", "status": "Standing", "lastUsed": "Last used", "lastError": "Last refusal", "expires": "Expires" }, "registry": { "title": "The registry", "new": "Add account", "empty": { "title": "No accounts yet", "body": "Add the Claude subscription or API key the raters should sign in with. Its secret is sealed and never shown again." }, "failed": { "title": "We could not load the registry" } }, "form": { "newTitle": "Add an LLM account", "editTitle": "Edit {{label}}", "label": "Label", "labelNote": "Whose it is and what it is, the way the registry should read it. Compared without regard to capitalization.", "provider": "How it signs in", "secret": "Token or key", "secretNote": "Pasted once and sealed. Nothing shows it again, not masked and not truncated.", "required": "An account needs a label, and a new one needs its token.", "register": "Add account", "saveFailed": "That did not save. Try again in a moment.", "about": "About LLM accounts", "newLede": "Register an account the raters can sign in with. Follow the steps on the right to mint its secret, then paste it in here.", "editLede": "Rename this account, or rotate its secret and the date it lapses. Leave the secret blank to keep the stored one.", "back": "LLMs", "providerFixed": "Fixed once registered, because the stored secret was sealed for this answer. Register a new account for a different one.", "billing": { "flat": "Billed flat", "perToken": "Billed per token" }, "replaceSecret": "Replace the token or key", "replaceSecretNote": "Leave this blank to keep the stored one. Anything here replaces it.", "expires": "Expires on", "expiresNote": "Optional. The date the secret stops working, if it has one. Every operator is warned before it lapses.", "save": "Save changes", "invalid": { "label": "Give the account a label, so the next operator knows whose it is.", "labelLength": "Keep the label under 120 characters.", "secret": "Paste the token or key itself.", "expires": "Choose a date that has not passed.", "staleExpiry": "The date recorded for the old secret has passed. Give the new one's date, or clear it if it does not expire." }, "notFound": "There is no LLM account with that id. It may never have been registered here.", "failed": "We could not load the account", "notFoundTitle": "No such account" }, "pin": { "model": "Model", "skills": "Skills version", "skillsNote": "Which uploaded version a pass runs. One whose upload never finished cannot be chosen.", "skillsCurrent": "Current", "skillsNotUploaded": "Listed, not uploaded", "skillsEmpty": "No version is uploaded yet. Upload one in step 1.", "skillsNoneUploaded": "Nothing was ever uploaded against any version stored here, so no pass could run one. Upload an archive for one first.", "skillsUnreadable": "We could not read the stored versions, so there is nothing to choose from. Reload the page.", "credential": "Signs in as", "credentialNote": "Which LLM account a pass uses. An account that cannot sign in cannot be chosen.", "cannotSignIn": "The pinned account cannot sign in right now, so nothing is being rated. Entries are still being accepted and are queueing up behind it. Fix that account or pin another one.", "save": "Save changes", "saveFailed": "That did not save. Try again in a moment.", "failed": { "title": "We could not load the pin" }, "start": "Start rating with these" }, "steps": { "done": "Done", "todo": "To do", "skills": { "title": "Rating skills", "description": "The rating skills from moore-skills, uploaded as one zip per version." }, "pin": { "title": "What rates entries", "description": "The model, skills version and account every new pass uses. A change applies to the next pass, never one already running." } }, "status": { "live": "Rating is live", "liveDetail": "{{model}} on skills {{skills}}, signed in as {{account}}.", "spend": "{{spent}} of {{ceiling}} tokens used today.", "notLive": "Rating is not live yet", "openLlms": "Open LLMs", "openUsage": "Open Usage", "blockers": { "accounts": "Add an LLM account on the LLMs page. Entries are refused until rating is set up.", "skills": "Upload a version of the rating skills in step 1. Entries are refused until rating is set up.", "pin": "Choose what rates entries in step 2. Entries are refused until rating is set up.", "deadAccount": "The account in step 2 cannot sign in, so nothing is being rated. Entries are queueing. Fix that account on the LLMs page or choose another in step 2.", "paused": "Today's token budget is spent, so rating is paused. Entries are queueing and it resumes on its own, or as soon as the daily budget is raised on the Usage page.", "expiredAccount": "The account in step 2 has passed its expiry, so nothing is being rated. Entries are queueing. Open it on the LLMs page and paste in a new secret with its new date, or choose another account in step 2.", "preflightFailed": "Reading the rubrics of the skills version in step 2 failed, so nothing is being rated. Entries are queueing. Read why beside it in step 1 and run it again, or upload and choose another version." } }, "skills": { "title": "Rating skills", "uploadTitle": "Upload skills", "version": "Version", "versionNote": "What every rating it produces records. Use a new one for every change; a version cannot be edited once used.", "source": "Source", "sourceNote": "Optional: where in moore-skills it came from, such as a branch or commit.", "archive": "Skills zip", "upload": "Upload", "uploadFailed": "That upload did not go through. Try again in a moment.", "inUse": "In use", "notUploaded": "Upload never finished", "remove": "Remove", "removeFailed": "That version could not be removed.", "preflight": { "pending": "Not read yet", "succeeded": "Read", "failed": "Failed", "runAgain": "Run again", "runAgainFailed": "That version's preflight could not be run again." }, "failed": "We could not load the rating skills", "empty": { "title": "No skills uploaded yet", "body": "Upload the rating skills zip from moore-skills to give the raters something to run." }, "columns": { "version": "Version", "source": "Source", "uploaded": "Uploaded", "preflight": "Rubric read", "ratings": "Ratings run" } }, "expiry": { "none": "No expiry", "on": "{{date}}", "expired": "Expired {{date}}", "soon_one": "In {{count}} day", "soon_other": "In {{count}} days" }, "instructions": { "title": "How to get it", "claudeSubscription": { "summary": "Rates on a Claude plan, billed flat. Every pass spends the plan's usage window, and the account reads as at its limit until the window resets.", "steps": { "setupToken": "On a machine where Claude Code is installed and signed in to the plan this should bill, run claude setup-token.", "approve": "It opens a browser to authorize. Approve it, then copy the token it prints. It begins sk-ant-oat01-.", "paste": "Paste the token into the token field on this page.", "expiry": "Set the expiry to one year from today. The token lives a year, and the date is what warns every operator before it lapses.", "label": "Give it a label naming whose plan it is, then save." }, "notes": { "rotate": "To rotate it, run claude setup-token again, open this account on the LLMs page, and paste the new token with its new date." } }, "anthropicApiKey": { "summary": "Rates on Anthropic's API, billed per token to the workspace the key belongs to.", "steps": { "console": "Open the Anthropic Console's API keys page in the workspace whose budget it should bill.", "create": "Choose Create Key and name it benchmark, so the console says where its spend goes.", "copy": "Copy the key. It begins sk-ant-api03- and is shown once.", "paste": "Paste it into the key field on this page, give it a label, and save." }, "notes": { "expiry": "Set an expiry only if the key was created with one. An invented date either warns about nothing or lapses without a word." } }, "openaiApiKey": { "summary": "An OpenAI API key, billed per token to the project it belongs to.", "steps": { "console": "Open OpenAI's API keys page in the project whose budget it should bill.", "create": "Choose Create new secret key and name it benchmark.", "copy": "Copy the key. It begins sk- and is shown once.", "paste": "Paste it into the key field on this page, give it a label, and save." }, "notes": { "expiry": "Set an expiry only if the key was created with one.", "claudeOnly": "Passes run Claude Code today, which cannot sign in with this key. Pinning it would leave every entry waiting unrated." } }, "geminiApiKey": { "summary": "A Gemini API key from AI Studio, billed per token to the Google Cloud project it belongs to.", "steps": { "studio": "Open AI Studio's API keys page, signed in as the account whose projects it should reach.", "create": "Choose Create API key and pick a Google Cloud project with billing enabled.", "copy": "Copy the key. It begins AIza.", "paste": "Paste it into the key field on this page, give it a label, and save." }, "notes": { "expiry": "Set an expiry only if the key was created with one.", "claudeOnly": "Passes run Claude Code today, which cannot sign in with this key. Pinning it would leave every entry waiting unrated." } }, "sealed": "The secret is sealed when it is saved. No screen or response shows it again, not masked and not truncated." } }, "activity": { "title": "Activity", "about": { "counts": "Sign-ups counts the accounts created each day, invited ones included. Entries counts every entry made, whatever became of it since. Ratings counts the passes that produced a rating an entrant can read, so an entry re-graded on appeal is rated on two days.", "refusals": "Refused uploads counts every entry the form or a connected agent's upload URL turned away: a broken or oversized archive, a refused image, a duplicate, the five a day, an account that could not enter, a sample that is not published, or a deployment not ready to rate. It cannot see an upload too large to reach the site at all, or an agent refused before it sent anything.", "days": "Each day is a UTC calendar day, read live: an account deleted since takes its sign-up and its entries with it. The two numbers at the top are the roster as it stands today, whatever range is shown." }, "roster": { "registered": "Registered", "registeredDefinition": "Every account on the deployment today, operators included.", "neverEntered": "Registered, never entered", "neverEnteredDefinition": "Accounts that have never made an entry. An entry later deleted still counts as entering." }, "series": { "sign_ups": "Sign-ups", "entries": "Entries", "refused": "Refused uploads", "ratings": "Ratings" }, "chart": { "label_one": "Sign-ups, entries, refused uploads and ratings on {{count}} day", "label_other": "Sign-ups, entries, refused uploads and ratings on each of {{count}} days" }, "refusals": { "title": "Refused uploads by kind", "badge": "{{kind}}: {{amount}}", "none": "No upload was refused in these days.", "kinds": { "not_pinned": "Not ready to rate", "unscreened": "Image check down", "account": "Account could not enter", "sample": "No such sample", "daily_limit": "Five a day", "archive": "Archive refused", "image": "Image refused", "duplicate": "Duplicate" } }, "tableLabel": "Activity by day", "columns": { "day": "Day (UTC)" }, "failed": "We could not load activity", "empty": { "title": "Nothing happened", "range": "Nobody signed up, entered, was refused or was rated in these days, so there is nothing to chart.", "never": "Nobody has signed up yet. The first account appears here." }, "clipped_one": "Showing the most recent {{count}} day. Accounts were created earlier than that.", "clipped_other": "Showing the most recent {{count}} days. Accounts were created earlier than that." }, "usage": { "title": "Usage", "about": { "tokens": "Tokens are everything rating spent: every pass, whether it wrote a rating or failed, every pre-scan of an entry, and every preflight of a skills version. That is exactly what the daily token budget counts.", "entries": "Entries rated counts the entries that received a rating an entrant can read that day. A pass that errored spent tokens and rated nothing, and an entry re-graded on appeal counts on each day it was rated.", "days": "Each day is a UTC calendar day. The budget is measured over a rolling 24 hours instead, so a day here and the budget's day rarely cover the same hours." }, "ranges": { "label": "Range", "week": "7 days", "month": "30 days", "all": "All time" }, "budget": { "title": "Daily token budget", "description": "What every rating pass together may spend in a rolling 24 hours. Raising it while rating is paused restarts it within a minute.", "spent": "Rating spent {{spent}} of {{ceiling}} tokens in the last {{hours}} hours.", "reserved_one": "{{reserved}} more is set aside for the pass still running, because a pass that has not finished has spent tokens and reported none.", "reserved_other": "{{reserved}} more is set aside for the {{count}} passes still running, because a pass that has not finished has spent tokens and reported none.", "ceiling": "Tokens a day", "ceilingNote": "What every rating pass together may spend in a rolling day. Tokens rather than money, because tokens are what a pass reports. It can be raised and lowered, never turned off.", "resumes": "Nothing can be claimed before {{when}}, which is when the oldest pass in the window ages out of it.", "save": "Save budget", "saveFailed": "That did not save. Try again in a moment.", "chips": { "rating": "Rating", "approaching": "Nearly spent", "paused": "Paused" }, "standings": { "rating": "There is room in the day for more passes.", "approaching": "{{remaining}} tokens left before the queue pauses.", "paused": "Rating is paused. Entries are still accepted and are queueing up, unspent: nothing is charged to anybody and no entry has failed. It starts again on its own as the window rolls, or as soon as the ceiling is raised." }, "failed": { "title": "We could not load the budget" } }, "chart": { "tokens": "Tokens", "entries": "Entries rated", "label_one": "Tokens spent and entries rated on {{count}} day", "label_other": "Tokens spent and entries rated on each of {{count}} days" }, "tableLabel": "Usage by day", "columns": { "day": "Day (UTC)", "tokens": "Tokens", "entries": "Entries rated", "rated": "Passes with a rating", "unrated": "Passes with no rating", "prescan": "Pre-scans", "preflight": "Preflights" }, "failed": "We could not load usage", "empty": { "title": "Nothing spent", "range": "No pass, pre-scan or preflight finished in these days, so there is nothing to chart.", "never": "Rating has not spent anything yet. The first pass that finishes appears here." }, "clipped_one": "Showing the most recent {{count}} day. Spending began earlier than that.", "clipped_other": "Showing the most recent {{count}} days. Spending began earlier than that." }, "audit": { "title": "Audit log", "lede": "What has been done to this deployment: curating the library, administering an account, and the security events an account records for itself. Newest first, and nothing here can be edited.", "system": "The system", "filters": { "actor": "Who", "actorPlaceholder": "Anybody. Search by name or email", "actorHint": "Type at least 3 characters to search accounts.", "noActors": "No account matches that.", "action": "What", "anyAction": "Anything" }, "actions": { "accountSuspended": "Account suspended", "accountReinstated": "Account reinstated", "accountSuspensionAmended": "Suspension reason changed", "accountDeleted": "Account deleted", "platformRolesChanged": "Operator role changed", "submissionRerun": "Entry run again", "submissionUploadStarted": "Entry upload started by an AI agent", "uploadImageRefused": "Upload refused by the image check", "avatarUploaded": "Profile picture uploaded", "avatarRemoved": "Profile picture removed", "avatarRefused": "Profile picture refused by the image check", "llmCredentialRegistered": "LLM account registered", "llmCredentialChanged": "LLM account changed", "ratingPinChanged": "What rates entries changed", "ratingBudgetChanged": "Daily token budget changed", "skillVersionUploaded": "Rating skills uploaded", "skillVersionChanged": "Rating skills edited", "skillVersionMadeCurrent": "Rating skills made current", "skillVersionRemoved": "Rating skills removed", "skillVersionPreflightRequested": "Rating skills preflight run again", "githubReportingSaved": "GitHub error reporting saved", "githubReportingRemoved": "GitHub token removed", "githubReportingTested": "Test issue filed", "slackWebhookSaved": "Slack webhook saved", "slackWebhookRemoved": "Slack webhook removed", "slackWebhookTested": "Slack test message posted", "slackWebhookKindsChanged": "Slack notices turned on or off", "passwordChanged": "Password changed", "mfaEnrolled": "Two-factor sign-in turned on", "mfaDisabled": "Two-factor sign-in turned off", "passkeyAdded": "Passkey added", "passkeyRemoved": "Passkey removed", "sessionRevoked": "Signed a session out", "sessionCreated": "Signed in", "created": "Created", "updated": "Edited", "destroyed": "Deleted", "invitationCreated": "Invitation sent", "invitationResent": "Invitation sent again", "invitationRevoked": "Invitation revoked", "invitationClaimed": "Invitation accepted", "passwordTemporarySet": "Reset to a temporary password", "reportDismissed": "Report dismissed", "reportActioned": "Report acted on", "reportRescanned": "Pre-scan run again", "submissionTakenDown": "Entry taken down", "submissionRescanned": "Entry sent back to the pre-scan", "commentTakenDown": "Comment taken down", "alertRecipientAdded": "Admin added to notifications", "alertRecipientEnabled": "Admin notifications turned on", "alertRecipientDisabled": "Admin notifications turned off", "alertRecipientChanged": "Admin notifications changed", "alertRecipientRemoved": "Admin removed from notifications", "oauthGranted": "AI agent connected", "oauthRevoked": "AI agent disconnected", "oauthCodeReused": "AI agent sign-in code replayed", "oauthRefreshReused": "AI agent token replayed", "feedbackReplied": "Feedback replied to", "feedbackClosed": "Feedback closed", "feedbackReopened": "Feedback reopened", "accountProfileUpdated": "Profile edited", "submissionCancelled": "Entry deleted by its entrant", "accountProfileContentRemoved": "Page content removed", "feedbackRepliesSecretMinted": "Email replies secret minted", "feedbackRepliesTurnedOff": "Email replies turned off" }, "subjects": { "account": "Account", "sample": "Sample", "sampleFile": "Sample file", "submission": "Entry", "gradingJob": "Rating pass", "skillVersion": "Rating skills version", "llmCredential": "LLM account", "ratingPin": "What rates entries", "ratingBudget": "Daily token budget", "githubReporting": "GitHub error reporting", "slackWebhook": "Slack webhook", "platformInvitation": "Invitation", "scorecardComment": "Comment", "report": "Report", "alertRecipient": "Admin", "oauthGrant": "AI agent connection", "entryUpload": "Entry upload link", "feedbackReplyWebhook": "Email replies" }, "columns": { "when": "When", "actor": "Who", "action": "What", "subject": "To what" }, "empty": { "title": "Nothing recorded yet", "body": "The log fills in as the library is curated and accounts are administered." }, "filteredEmpty": { "title": "Nothing matches these filters", "body": "Nobody has done that yet. Widen who or what to see more of the log." }, "failed": { "title": "We could not read the audit log" } }, "hub": { "title": "Admin settings", "samples": "Add samples, upload the files each one ships, and publish them to the library.", "entries": "Every entry, its state and what rating it cost, and running one again.", "entrants": "Everybody registered, suspending an account, and granting the operator role.", "activity": "Sign-ups, entries, refused uploads and ratings each day, charted, and how many accounts never entered.", "raters": "Rating skills, and what rates entries.", "usage": "Tokens spent and entries rated each day, charted, and the daily token budget.", "llms": "The Claude subscriptions and API keys rating passes sign in with, and when each one lapses.", "audit": "What has been done to this deployment, newest first, filtered by actor and action.", "github": "The GitHub repository this deployment files its own failures in, and the token it files them with.", "slack": "The Slack webhook that tells #benchmark-admin what is happening, and which kinds of notice it posts.", "groups": { "library": "Library and entries", "people": "People", "system": "Rating, errors, Slack and Jira", "moderation": "Reported content" }, "reports": "Entries the pre-scan held, and scorecards, comments and contestants' pages people reported, each dismissed or taken down.", "alerts": "The admins emailed when something needs one of them, and what each one hears about.", "feedback": "What entrants wrote to us, answering it, and closing a conversation once it is done.", "jira": "The Jira site known bugs are read from and new ones filed in, and the API token that signs in to it.", "emailReplies": "Answering a feedback email by replying to it: the mail records, Brevo's webhook, and the secret it presents.", "emailDelivery": "The addresses Brevo stopped delivering to, the webhook that says so, and unblocking one." }, "checklist": { "steps": "Steps", "notes": "Good to know" }, "github": { "title": "Error reporting", "lede": "Anything that fails here, a request that crashed, a page that broke in somebody's browser, a rating pass that fell over, is filed as an issue in one GitHub repository. A fault that happens again is counted against the issue it already has rather than filed twice.", "failed": "We could not load the error reporting settings", "status": { "title": "Status", "unconfigured": "No token is saved, so nothing is filed. Faults are still counted, and the first one after a token is saved is filed then.", "repository": "Repository", "expires": "Expires", "noExpiry": "No date given", "expired": "Expired", "expiresSoon": "Expires soon", "lastUsed": "Last used", "never": "Never", "lastError": "Last refusal", "noError": "None" }, "form": { "title": "Token", "repository": "Repository", "repositoryDescription": "Where issues are filed, written as owner/name.", "token": "Personal access token", "tokenDescription": "Sealed when it is saved, and never shown again.", "tokenKeepDescription": "A token is saved. Leave this blank to keep it, or paste a new one to replace it.", "expires": "Expiration date", "expiresDescription": "Optional. Copy it from GitHub: nothing can read it off the token, and it is the only warning anybody gets before filing stops.", "save": "Save", "saved": "Saved.", "incomplete": "Fill in the repository and the token first.", "refused": "The settings could not be saved. Try again in a moment.", "invalid": { "repository": "Write the repository as owner/name.", "token": "Paste a personal access token.", "expires": "Choose a date that has not passed." } }, "test": { "action": "Send test issue", "sent": "Test issue filed:", "issue": "issue #{{number}}", "refused": "The test issue could not be filed." }, "remove": { "action": "Remove token", "confirm": "Remove it", "cancel": "Keep it", "consequence": "Nothing is filed until a new token is saved. Faults are still counted.", "refused": "The token could not be removed. Try again in a moment." }, "replace": { "action": "Enter a new token", "cancel": "Keep the current token" }, "reports": { "title": "Recent faults", "empty": "Nothing has failed yet.", "occurrences_one": "Happened {{count}} time", "occurrences_other": "Happened {{count}} times", "lastSeen": "Last seen {{when}}", "issue": "Open the issue", "noIssue": "No issue yet", "origins": { "server": "Server", "browser": "Browser", "worker": "Rating worker" }, "states": { "queued": "Filing", "open": "Filed", "failed": "Filing failed", "withheld": "Waiting on the hourly limit", "unconfigured": "Counted, no token saved" } }, "checklist": { "title": "Minting the token", "summary": "A fine-grained personal access token that can do one thing: read and write issues in one repository.", "steps": { "open": "Open GitHub's fine-grained token page, which is Settings, Developer settings, Personal access tokens, Fine-grained tokens, Generate new token.", "name": "Name it after this deployment, for example benchmark-error-reporting, so whoever reads the token list knows what it is for.", "owner": "Set Resource owner to the account or organization that owns the repository issues should go to. A token owned by your personal account cannot reach an organization's repository.", "expiry": "Choose an expiration and note the date. It goes in the form here too.", "repository": "Under Repository access, choose Only select repositories and pick exactly the one repository.", "permissions": "Under Repository permissions, set Issues to Read and write. Metadata is added as read-only on its own. Grant nothing else, and leave every organization permission at no access.", "generate": "Choose Generate token and copy it. GitHub shows it once.", "paste": "Write the repository as owner/name, paste the token, enter the expiration date, and save.", "test": "Choose Send test issue. An issue appearing in the repository is the proof that the token can file." }, "notes": { "sealed": "The token is sealed before it is stored, and no screen or endpoint shows it again. Pasting a new one over it is how it is rotated.", "approval": "An organization can require an owner to approve fine-grained tokens. Until one does, GitHub refuses the token, and the test issue says so.", "dedupe": "Issues are de-duplicated: a fault that happens again is counted against the issue it already has, and closing an issue lets the next occurrence open a fresh one.", "unconfigured": "With no token saved nothing is filed, but every fault is still counted, so the list below fills either way." } } }, "slack": { "title": "Slack", "lede": "Leadership and operators hear about everything that matters in one Slack channel, #benchmark-admin: sign-ups, entries, ratings, failures, deletions, reports, feedback, the raters, the budget, expiring secrets and the GitHub issues this site files. A notice says what happened and links to it, and never quotes what anybody wrote. It is a second place beside the admin notification emails, not a replacement for them.", "failed": "We could not load the Slack settings", "status": { "title": "Status", "unconfigured": "No webhook is saved, so nothing is posted to Slack. The admin notification emails carry on either way.", "state": "Standing", "posting": "Posting", "revoked": "Revoked", "expires": "Rotate by", "noExpiry": "No date given", "expired": "Past its date", "expiresSoon": "Due soon", "lastUsed": "Last posted", "never": "Never", "lastError": "Last refusal", "noError": "None" }, "form": { "title": "Webhook", "url": "Webhook URL", "urlDescription": "Sealed when it is saved, and never shown again.", "urlKeepDescription": "A webhook is saved. Leave this blank to keep it, or paste a new one to replace it.", "expires": "Rotate by", "expiresDescription": "Optional. Slack webhooks do not lapse on their own, so this is a reminder: operators are warned two weeks ahead.", "save": "Save", "saved": "Saved.", "incomplete": "Paste the webhook URL first.", "refused": "The webhook could not be saved. Try again in a moment.", "invalid": { "missing": "Paste the webhook URL.", "url": "A Slack webhook URL starts with https://hooks.slack.com/services/ and has three parts after it.", "expires": "Choose a date that has not passed." } }, "test": { "action": "Send test message", "sent": "Test message posted. Look for it in the channel.", "refused": "The test message could not be posted." }, "remove": { "action": "Remove webhook", "confirm": "Remove it", "cancel": "Keep it", "consequence": "Nothing is posted to Slack until a new webhook is saved. The admin notification emails carry on.", "refused": "The webhook could not be removed. Try again in a moment." }, "checklist": { "title": "Making the webhook", "summary": "A Slack app with Incoming Webhooks and nothing else, posting to one channel.", "steps": { "channel": "In Slack, create the channel operators read, #benchmark-admin, if it does not exist yet.", "open": "Open Slack's app page and choose Create New App, then From a manifest.", "manifest": "Pick the workspace, paste this manifest, and create the app. A workspace admin may have to approve it.", "webhooks": "Open Incoming Webhooks, turn it on if it is off, and choose Add New Webhook to Workspace.", "choose": "Choose #benchmark-admin and allow it. Slack shows a URL starting https://hooks.slack.com/services/.", "paste": "Copy the URL, paste it here, choose a date to rotate it by if you want a reminder, and save.", "test": "Choose Send test message. A message appearing in the channel is the proof that the webhook can post." }, "notes": { "sealed": "The URL is the whole credential. It is sealed before it is stored, and no screen or endpoint shows it again. Pasting a new one over it is how it is rotated.", "scope": "A webhook can post to one channel and do nothing else, which is why this is a webhook rather than a bot token: a leaked URL is noise in one channel, not access to the workspace.", "content": "A notice names the sample and the entrant, a score, the kind of report and its category, who wrote feedback, an account's label, the budget's numbers, or the issue filed. It never carries what anybody wrote, anything from an archive, an email address or a secret.", "cap": "Each kind posts at most ten notices an hour. Past that, one message says more of that kind are waiting, and the rest are in the console, so a busy kind never silences another. The rater problems, the budget and expiring secrets are never held back. The emails are counted separately.", "revoked": "If Slack answers that the webhook is gone, this page says so and nothing more is posted until a new one is saved. If the channel was only archived or the app restricted, undo that in Slack and send a test message: one that posts puts the webhook back in use." } }, "kinds": { "title": "What posts", "summary": "Every kind posts until it is turned off here. Turning one off takes effect from the next notice.", "refused": "That change did not go through. Try again." } }, "levers": { "grantOperator": { "title": "Make {{name}} an operator?", "body": "{{name}} ({{email}}) will be able to suspend and reinstate anybody, appoint and remove other operators, read every account's email address and history, curate the sample library, and change what rates entries and what rating may spend. Only do this for somebody who runs this deployment.", "confirm": "Make operator" }, "revokeOperator": { "title": "Remove {{name}} as an operator?", "body": "{{name}} ({{email}}) loses the console at once, including this screen. Their own account, entries and standing on the leaderboard are untouched. Another operator can appoint them again later.", "confirm": "Remove operator" }, "reinstate": { "title": "Lift the suspension on {{name}}?", "body": "{{name}} ({{email}}) can enter and appeal again, and their rows return to the leaderboard. The suspension and this reinstatement both stay in the audit log.", "confirm": "Lift suspension" }, "gone": "That did not change. The account may no longer exist.", "failed": "That could not be done. Try again in a moment." }, "entrant": { "notFound": "There is no account at this address.", "failed": { "title": "We could not load this account" }, "verified": "Email verified", "unverified": "Email not verified", "suspendedBecause": "Suspended: {{reason}}", "tabs": { "label": "About this account", "activity": "Activity", "entries": "Entries", "page": "Public page" }, "activity": { "limit_one": "The most recent {{count}} event, newest first.", "limit_other": "The most recent {{count}} events, newest first.", "failed": "We could not load this account's activity", "empty": { "title": "Nothing recorded yet", "body": "Sign-ins, account changes, entries and ratings appear here as they happen." }, "signIn": { "registration": "Registered and signed in", "password": "Signed in with a password", "secondFactor": "Signed in with a password and a second factor", "emailCode": "Signed in with an emailed code", "passkey": "Signed in with a passkey", "oauth": "Signed in through a single sign-on provider", "unknown": "Signed in", "invitation": "Signed in on accepting an invitation" }, "roleGranted": "Made an operator", "roleRevoked": "Removed as an operator", "by": "By {{name}}", "reason": "Reason: {{reason}}", "entrySubmitted": "Entered {{sample}}", "appealLodged": "Appealed the rating of {{sample}}", "rating": { "success": "{{sample}} was rated", "partial": "{{sample}} was partly rated", "nothingToRate": "Nothing in {{sample}} could be rated", "errored": "A rating pass over {{sample}} failed", "scored": { "initial": "First pass, scoring {{score}}", "appeal": "Re-grade after an appeal, scoring {{score}}", "operatorRerun": "Run again by an operator, scoring {{score}}" }, "unscored": { "initial": "First pass", "appeal": "Re-grade after an appeal", "operatorRerun": "Run again by an operator" } }, "invitation": { "created": "Invited {{email}}", "resent": "Sent the invitation to {{email}} again", "revoked": "Revoked the invitation to {{email}}", "claimed": "Accepted an invitation and opened this account", "unknownAddress": "an address that was not recorded" } }, "origin": { "reportedLocation": "Location, as the browser reported it", "ipAddress": "IP address", "browser": "Browser", "notRecorded": "Not recorded", "unknown": "Origin not recorded" }, "entries": { "failed": "We could not load this account's entries", "empty": { "title": "No entries yet", "body": "Entries this account makes appear here, each opening its scorecard." } }, "page": { "public": "Public", "private": "Private", "publicHint": "Anybody can read this page, picture, bio and links included.", "privateHint": "Only the name shows to anybody else. You see everything on it because you operate this deployment.", "open": "Open their page", "empty": { "title": "Nothing on this page", "body": "They have not added a picture, a bio or any links." }, "bio": "Bio", "links": "Links", "picture": "Picture", "noBio": "No bio.", "noLinks": "No links.", "noPicture": "No picture.", "pictureAlt": "{{name}}'s picture" } }, "invite": { "action": "Invite", "title": "Invite somebody", "lede_one": "They get an email with a link that works for {{count}} hour. Following it, they choose a password, and their account exists with this address already confirmed. Then they give their name like anybody who signs up.", "lede_other": "They get an email with a link that works for {{count}} hours. Following it, they choose a password, and their account exists with this address already confirmed. Then they give their name like anybody who signs up.", "roleLabel": "They join as", "roles": { "contestant": "Enters the benchmark under their own name, with no operator powers.", "operator": "Runs this deployment from the moment they accept: every account's address and history, suspending and appointing people, the sample library, and what rating may spend." }, "required": "Enter the address to invite.", "confirmContestant": "Send invitation", "confirmOperator": "Invite as an operator", "failed": "The invitation could not be sent. Try again in a moment." }, "invitations": { "title": "Pending invitations", "pages": "Pages of pending invitations", "expired": "Expired", "columns": { "email": "Address", "role": "Joins as", "invitedBy": "Invited by", "sent": "Sent", "expires": "Link expires" }, "menu": { "label": "Actions for the invitation to {{email}}", "resend": "Resend", "revoke": "Revoke" }, "levers": { "resend": { "title": "Resend the invitation to {{email}}?", "body_one": "A new email goes to {{email}} with a fresh link that works for {{count}} hour. The link they already have stops working at once.", "body_other": "A new email goes to {{email}} with a fresh link that works for {{count}} hours. The link they already have stops working at once.", "confirm": "Resend" }, "revoke": { "title": "Revoke the invitation to {{email}}?", "body": "The link in their inbox stops working, and no account is created. You can invite {{email}} again later.", "confirm": "Revoke invitation" }, "gone": "That did not change. The invitation may no longer exist." }, "empty": { "title": "Nobody is waiting on an invitation", "body": "Invite somebody and they appear here until they accept, or until you revoke it." }, "failed": { "title": "We could not load the pending invitations" } }, "temporaryPassword": { "confirm": { "title": "Reset {{name}} to a temporary password?", "body": "{{name}} ({{email}}) is signed out of every browser at once, and their current password stops working. You are shown a temporary password one time, to give them yourself. At their next sign-in they must choose a new password before they can do anything else.", "action": "Reset password" }, "result": { "title": "Temporary password for {{name}}", "label": "Temporary password", "copy": "Copy the password", "copied": "Copied", "once": "This is the only time it is shown. Copy it now and pass it on by a channel you trust. It is not stored, emailed, or written to the audit log." } }, "reports": { "title": "Reported content", "lede": "Entries the pre-scan held before rating, and scorecards, comments and contestants' pages people reported. Dismissing a report leaves what it reported as it is, and dismissing a held entry's report queues its rating, and running its pre-scan again asks the pre-scan once more. Taking one down removes what it reported and closes every report about it; for a page, you choose which of its bio, links and picture go.", "tabs": { "label": "Kind of report", "entries": "Entries", "scorecards": "Scorecards", "comments": "Comments", "profiles": "Profiles" }, "statuses": { "label": "Show", "open": "Unreviewed", "all": "Every report" }, "statusLabels": { "open": "Unreviewed", "dismissed": "Dismissed", "actioned": "Taken down", "rescanned": "Scanned again" }, "columns": { "target": "Reported", "source": "Reported by", "reason": "Why", "when": "When", "status": "Review" }, "sources": { "prescan": "Automated pre-scan", "wordlist": "Abusive words" }, "categories": { "harassment": "Harassment", "hate": "Hate", "sexual": "Sexual content", "violence": "Violence", "self_harm": "Self-harm", "malware": "Malware", "spam": "Spam", "not_collateral": "Not verification collateral", "other": "Something else" }, "wordlistReason": "Matched the abusive-words list when it was written.", "unnamed": "An account with no name", "deletedReporter": "A deleted account", "target": { "by": "by {{name}}", "openComments": "Open the comments", "gone": "No longer exists", "pageParts": { "bio": "Bio", "links_one": "{{count}} link", "links_other": "{{count}} links", "picture": "Picture" }, "pageEmpty": "Nothing on the page now" }, "menu": { "label": "Actions for this report", "dismiss": "Dismiss", "rescan": "Run the pre-scan again", "takeDown": "Take down", "account": "Open their account", "removePageContent": "Remove page content" }, "review": { "dismissTitle": "Dismiss this report?", "dismissBody": "The report closes and what it reported stays exactly as it is.", "dismissHeldBody": "The report closes and the entry is queued for rating without being pre-scanned again.", "rescanTitle": "Run the pre-scan again?", "rescanBody": "The report closes and the entry goes back to the pre-scan, which runs once. If it clears the entry, the entry is rated. If it flags it or fails again, the entry is held here with a new report.", "takeDownEntryTitle": "Take this entry down?", "takeDownEntryBody": "It leaves the leaderboard, its public scorecard closes, and nothing rates it again. Its entrant is shown your reason, and every other report about it closes too.", "takeDownCommentTitle": "Remove this comment?", "takeDownCommentBody": "It is removed the way its author would remove it, and replies under it stay. Every other report about it closes too.", "note": "Note for the log", "noteForEntrant": "Reason shown to the entrant", "noteRequired": "Say why. The entrant is shown it.", "dismiss": "Dismiss report", "rescan": "Run the pre-scan again", "takeDown": "Take it down", "removeComment": "Remove comment", "failed": "That review did not go through. Try again." }, "emptyOpen": { "title": "Nothing to review", "body": "Nothing on this tab is waiting for a decision. Reports arrive here as people file them and as the pre-scan holds entries." }, "emptyAll": { "title": "Nothing reported yet", "body": "Nothing of this kind has ever been reported." }, "failed": { "title": "We could not load the reports" }, "pages": "Pages of reports", "prescanFailed": "Pre-scan failed", "failures": { "unreadable": "The model answered with something other than a verdict.", "unanswered": "The pre-scan ended without an answer.", "overran": "The pre-scan ran too long and was stopped.", "overgrown": "The pre-scan's workspace grew too large and was stopped." } }, "alerts": { "title": "Admin notifications", "lede": "The admins this site emails when something needs one of them, and what each of them hears about. One email goes out per event, with every admin who hears about that kind on it, so everybody can see who else was told. Reports, feedback and failed entries are each emailed at most {{limit}} times an hour; past that, one note says more are waiting. Everything here also posts to #benchmark-admin in Slack.", "add": "Add admin", "addTitle": "Email an admin when something needs them", "addLede": "A shared inbox works as well as a person. They hear about every kind until you choose otherwise.", "email": "Email address", "name": "Name (optional)", "kinds": "Emails about", "kindsHint": "Choose at least one. To stop emailing somebody for a while, turn them off instead.", "addConfirm": "Add admin", "addFailed": "That admin could not be added. Try again.", "toggleFailed": "That change did not go through. Try again.", "columns": { "who": "Admin", "kinds": "Hears about", "status": "Emails", "added": "Added" }, "statuses": { "on": "On", "off": "Off" }, "menu": { "label": "Actions for {{email}}", "edit": "Choose what they hear", "enable": "Turn on", "disable": "Turn off", "remove": "Remove" }, "edit": { "title": "What {{email}} hears about", "confirm": "Save", "failed": "That change could not be saved. Try again." }, "remove": { "title": "Stop emailing {{email}}?", "body": "They stop being emailed about anything. Turning them off instead keeps them here for when they are wanted again.", "confirm": "Remove admin", "failed": "That admin could not be removed. Try again." }, "empty": { "title": "Nobody is emailed", "body": "Until an admin is added here, nobody is emailed about reports, feedback, failed entries, the raters, the budget or expiring secrets. Every one of them still waits on the console, and posts to Slack." }, "failed": { "title": "We could not load the admin list" } }, "feedback": { "title": "Feedback", "lede": "Every conversation an entrant started, most recent activity first. Open one to reply: the entrant is emailed a link to it, never the reply itself, and writing feedback emails them nothing. Entrants have no list of their own, so the email is how they come back.", "awaiting_one": "{{count}} awaiting a reply", "awaiting_other": "{{count}} awaiting a reply", "statuses": { "label": "Show", "open": "Open", "closed": "Closed", "all": "All" }, "search": { "placeholder": "Search", "tooShort": "Type at least {{count}} characters to search." }, "columns": { "conversation": "Conversation", "category": "Category", "author": "From", "entry": "Entry", "status": "Status", "messages": "Messages", "last": "Last message" }, "noEntry": "None", "failed": "We could not load the feedback", "pages": "Feedback pages", "empty": { "title": "No feedback yet", "body": "Nobody has written to us. A conversation appears here the moment somebody does." }, "emptyOpen": { "title": "Nothing open", "body": "Every conversation is closed. New feedback appears here the moment somebody writes." }, "emptyClosed": { "title": "Nothing closed", "body": "No conversation has been closed yet." }, "emptySearch": { "title": "No conversation matches", "body": "Nothing written by or to that name, address or phrase. Try fewer words." } }, "emailReplies": { "title": "Email replies", "lede": "An entrant can answer the email telling them about a reply by replying to it, and what they write lands in the conversation as theirs, without the email it quotes or their signature. Brevo receives the email and posts it here, presenting a secret this page mints. While this is off, replies to that email reach the company inbox instead.", "failed": "We could not load the email reply settings", "status": { "title": "Status", "off": "Off. Reply emails name no reply address, so answering one reaches the company inbox rather than the conversation.", "on": "On", "standing": "Standing", "since": "Turned on", "minted": "Secret minted", "lastReceived": "Last delivery from Brevo", "never": "Never" }, "turnOn": { "action": "Turn on", "refused": "Replying by email could not be turned on. Try again in a moment." }, "rotate": { "action": "Mint a new secret", "confirm": "Mint it", "cancel": "Keep the current one", "consequence": "The current secret stops working at once. Give Brevo the new one straight away, or every reply is refused until you do.", "refused": "A new secret could not be minted. Try again in a moment." }, "turnOff": { "action": "Turn off", "confirm": "Turn it off", "cancel": "Keep it on", "consequence": "Reply emails stop naming a reply address, and anything Brevo posts is refused.", "refused": "Replying by email could not be turned off. Try again in a moment." }, "secret": { "title": "Copy the secret now", "body": "It is not shown again. Brevo presents it as a bearer token on every post, and the command below gives it to Brevo.", "secretLabel": "Secret", "commandLabel": "Create Brevo's inbound webhook with it", "rotateHint": "Rotating? Update the webhook Brevo already has instead: the same body, sent with PUT to /v3/webhooks/ and that webhook's id." }, "checklist": { "title": "Setting it up", "summary": "Mail to {{domain}} is received by Brevo's inbound parsing and posted to this site. These are the steps outside it.", "steps": { "plan": "Check that the Brevo account's plan includes Inbound parsing. It is set up through Brevo's API rather than its screens.", "mx": "Add two MX records on {{domain}}: inbound1.sendinblue.com at priority 10 and inbound2.sendinblue.com at priority 20. In Namecheap the host is {{host}}.", "mint": "Press Turn on here, or Mint a new secret, and keep the page open: the secret is shown once.", "webhook": "Run the command shown beside the secret with this deployment's Brevo API key. It tells Brevo to post the mail {{domain}} receives to this site, presenting the secret.", "test": "From the address of an account that wrote feedback, reply to a reply email, and check the reply lands in that conversation." }, "notes": { "matched": "A reply is posted only from its conversation's author, at the address the account has now, into an open conversation, under the same daily allowance a typed message has. Anything else is dropped and logged, never posted.", "trimmed": "The email it quotes, a signature and any attachment are left out. A reply over 5,000 characters is cut at 5,000.", "rotation": "Minting a new secret stops the old one at once, so Brevo's webhook needs the new one straight after." } } }, "emailDelivery": { "title": "Email delivery", "lede": "Brevo stops delivering to an address when its owner presses the Unsubscribe button a mail client draws on our emails, when it bounces for good, or when it is marked as spam, and from then on every email to it, sign-in codes included, is accepted and never delivered. Brevo tells this site through a webhook, presenting a secret this page mints, and the addresses it named are listed below. Nothing here unblocks an address on its own.", "failed": "We could not load the email delivery settings", "status": { "title": "Webhook", "off": "Off. Brevo is not telling this site which addresses it stopped delivering to.", "on": "On", "standing": "Standing", "since": "Turned on", "minted": "Secret minted", "lastReceived": "Last delivery from Brevo", "never": "Never" }, "turnOn": { "action": "Turn on", "refused": "The webhook could not be turned on. Try again in a moment." }, "rotate": { "action": "Mint a new secret", "confirm": "Mint it", "cancel": "Keep the current one", "consequence": "The current secret stops working at once. Give Brevo the new one straight away, or every event it posts is refused until you do.", "refused": "A new secret could not be minted. Try again in a moment." }, "turnOff": { "action": "Turn off", "confirm": "Turn it off", "cancel": "Keep it on", "consequence": "Anything Brevo posts is refused, and the addresses already listed stay listed.", "refused": "The webhook could not be turned off. Try again in a moment." }, "secret": { "title": "Copy the secret now", "body": "It is not shown again. Brevo presents it as a bearer token on every post, and the command below gives it to Brevo.", "secretLabel": "Secret", "commandLabel": "Create Brevo's transactional webhook with it", "rotateHint": "Rotating? Update the webhook Brevo already has instead: the same body, sent with PUT to /v3/webhooks/ and that webhook's id." }, "checklist": { "title": "Setting it up", "summary": "Brevo's transactional webhook posts here whenever it stops delivering to an address. These are the steps outside this site.", "steps": { "mint": "Press Turn on here, or Mint a new secret, and keep the page open: the secret is shown once.", "webhook": "Run the command shown beside the secret with this deployment's Brevo API key. It subscribes {{url}} to the hard bounce, blocked, spam and unsubscribed events.", "test": "Send a sign-in code to an address you control, unsubscribe it with the button in your mail client, and check the address appears below." }, "notes": { "never": "Nothing unblocks an address on its own. Anybody can ask for a sign-in code to any address, so an automatic unblock would re-subscribe somebody who unsubscribed to stop exactly that.", "blocked": "Brevo says blocked every time it declines to send to an address already on its blocklist, so one address can be reported many times.", "rotation": "Minting a new secret stops the old one at once, so Brevo's webhook needs the new one straight after." } }, "addressesTitle": "Addresses Brevo stopped delivering to", "addressesFailed": "We could not load the addresses", "count_one": "{{count}} address and reason recorded.", "count_other": "{{count}} addresses and reasons recorded.", "pages": "Pages of addresses", "noKey": "This deployment holds no Brevo key, so an address cannot be unblocked from here. Unblock it in Brevo's transactional blocklist.", "empty": { "title": "Brevo is delivering to every address", "body": "When Brevo stops delivering to one, it is listed here with why." }, "tableLabel": "Addresses Brevo stopped delivering to", "columns": { "email": "Address", "account": "Account", "reason": "Why", "last": "Last reported", "reports": "Reports", "action": "Unblock" }, "reasons": { "unsubscribed": "Unsubscribed", "hard_bounce": "Hard bounce", "spam": "Marked as spam", "blocked": "Blocked" }, "unnamed": "An account without a name", "noAccount": "No account", "unblock": { "action": "Unblock", "title": "Unblock {{email}}?", "body": "Brevo will deliver every email to this address again, sign-in codes and replies included. If its owner unsubscribed on purpose, they will start hearing from us again, so unblock only an address whose owner asked for it.", "confirm": "Unblock it", "failed": "The address could not be unblocked. Try again in a moment." }, "banner": { "title": "Brevo stopped delivering to this address", "body": "Emails to it, sign-in codes included, are accepted and never arrive.", "open": "Email delivery" } }, "remove": { "title": "Remove {{name}}'s entry on {{sample}}?", "titleUnnamed": "Remove this entry on {{sample}}?", "lede": "Removing takes the entry down. A pass waiting on it never runs and a pass running on it stops, it leaves the leaderboard, its public scorecard closes, and its archive is deleted. The entrant is shown your reason. This cannot be undone.", "reason": "Why it is being removed", "required": "Say why. The entrant is shown the reason.", "confirm": "Remove entry", "failed": "That could not be removed. Try again in a moment." }, "pageRemoval": { "title": "Remove from {{name}}'s page?", "lede": "Choose what goes. {{name}} keeps competing: nobody is suspended, their entries stay on the board and their comments stay where they are. They are shown your reason in their account settings, and can put something new in its place.", "partsLabel": "What goes", "parts": { "bio": "The bio", "links": "Every link", "picture": "The picture" }, "notOnPage": "Not on the page", "reason": "Reason shown to them", "reasonHint": "They read it in their account settings, with the date. Your name is not shown.", "reasonRequired": "Say why. They are shown it.", "chooseRequired": "Choose what to remove.", "confirm": "Remove", "failed": "That removal did not go through. Try again." }, "jira": { "title": "Jira", "lede": "Every entry's bug_report.md is scored against the RTL bugs already known for its sample, which live in one Jira project, SAMPLES unless you say otherwise. This page holds the site, the account and the API token the backend reads them with, and files new bugs under for a person to review. Nothing an entrant sees ever names a Jira key, and the raters never hold this token.", "failed": "We could not load the Jira connection", "status": { "title": "Status", "unconfigured": "No connection is saved, so entries are not scored against known bugs. Every other score is unaffected.", "site": "Site", "email": "Account", "project": "Project", "expires": "Expires", "expired": "Expired", "expiresSoon": "Expires soon", "lastUsed": "Last used", "never": "Never", "lastError": "Last refusal", "noError": "None" }, "edit": { "action": "Edit connection", "cancel": "Keep the current connection" }, "form": { "title": "Connection", "siteUrl": "Site URL", "siteUrlDescription": "The address you open Jira at, with nothing after .atlassian.net.", "email": "Account email", "emailDescription": "The Atlassian account the token belongs to. Bugs are filed under its name.", "projectKey": "Project key", "projectKeyDescription": "The project the known bugs live in.", "token": "API token", "tokenDescription": "Sealed when it is saved, and never shown again.", "tokenKeepDescription": "A token is saved. Leave this blank to keep it, or paste a new one to replace it.", "expires": "Expires on", "expiresDescription": "The date Atlassian showed when the token was made. Operators are warned two weeks ahead.", "save": "Save", "saved": "Saved.", "incomplete": "Fill in every field first.", "refused": "The connection could not be saved. Try again in a moment.", "invalid": { "siteUrl": "A Jira Cloud site looks like https://your-site.atlassian.net, with nothing after it.", "email": "Enter the email address of the Atlassian account.", "projectKey": "A project key is two to ten capital letters, digits or underscores, starting with a letter, such as SAMPLES.", "token": "Paste the API token.", "expiresMissing": "Choose the date the token expires.", "expires": "Choose a date that has not passed." } }, "test": { "action": "Test connection", "passed": "Connected.", "failed": "Not working.", "refused": "The connection could not be tested." }, "remove": { "action": "Remove connection", "confirm": "Remove it", "cancel": "Keep it", "consequence": "New entries are not scored against known bugs until a connection is saved again. Results already given stay as they are.", "refused": "The connection could not be removed. Try again in a moment." }, "checklist": { "title": "Making the token", "summary": "An Atlassian API token for an account that can read the project and create bugs in it.", "steps": { "account": "Sign in to Atlassian as the account bugs should be filed under. It needs to browse the project and create issues in it.", "open": "Open Atlassian's API token page and choose Create API token.", "create": "Name it MooresLab Benchmark and choose when it expires. Note the date: it goes in the form here.", "copy": "Copy the token. Atlassian shows it once.", "detectedBy": "In Jira, make sure the Bug field Detected by has an option named Benchmark. New bugs are filed with it, and filing fails without it.", "labels": "Agree the labels with whoever reviews bugs: needs-review while a bug waits, swapped for rejected to reject it, and removed to approve it.", "paste": "Paste the site, the account email, the project key, the token and its expiry date here, and save.", "test": "Choose Test connection. It signs in and reads the project's Bug fields, and writes nothing to Jira." }, "notes": { "sealed": "The token is sealed before it is stored, and no screen or endpoint shows it again. Saving a new one over it is how it is rotated.", "reads": "For each entry, the backend reads the sample's known bugs (the issues whose IP is the sample's slug) and hands them to the scoring skill in a closed workspace. The raters never see them.", "files": "A new bug the skill confirmed in the RTL is filed with the label needs-review and Detected by set to Benchmark. It earns credit only once a person approves it.", "review": "Review lives in labels, never statuses, so the board's statuses stay the team's own.", "unconfigured": "With no connection saved, entries are not scored against known bugs, and every other score is unaffected." } } }, "notificationKinds": { "entries": { "label": "New entries", "description": "Somebody entered: the sample and who entered it." }, "ratings": { "label": "Ratings", "description": "An entry was rated: the sample, who entered it, and the score." }, "deletions": { "label": "Deleted entries", "description": "An entrant deleted one of their entries, rated or not." }, "sign_ups": { "label": "Sign-ups", "description": "Somebody set up an account: their name, never their email address." }, "failures": { "label": "Failed entries", "description": "An entry could not be rated: its attempts ran out, or its archive could not be unpacked." }, "raters": { "label": "Rater problems", "description": "A preflight failed, or the vendor refused an LLM account or ran it dry. Rating may have stopped." }, "budget": { "label": "Budget", "description": "Rating passed 80% of the daily token budget, or paused at it." }, "secrets": { "label": "Expiring secrets", "description": "An LLM account, the GitHub token, the Slack webhook or the Jira token lapses in 7, 3, 2 or 1 days or 12 hours, or has lapsed." }, "reports": { "label": "Reports", "description": "Something was reported, by a person, the pre-scan or the abusive-words list." }, "feedback": { "label": "Feedback", "description": "An entrant wrote to us, starting a conversation or adding to one." }, "error_issues": { "label": "Filed issues", "description": "This site filed a GitHub issue about a failure of its own." }, "bugs_found": { "label": "Bugs found", "description": "An entry's bug report was scored: the sample and the counts, never a bug's details." } } }, "errorBoundary": { "title": "Something broke", "body": "This page hit an error it could not recover from on its own. It has been reported. Trying again often works; if it does not, reloading starts the site afresh.", "retry": "Try again", "reload": "Reload the site" }, "framedExit": { "title": "This page opens in its own tab", "body": "Signing in, entering and your account are on the MooresLab Benchmark itself, not inside another site's page.", "open": "Open in a new tab" }, "errorMargin": { "note_unmeasured": "This is rated by an AI system, so two passes over the same collateral do not score identically. The margin has not been measured yet.", "note_unmeasuredEntering": "This is rated by an AI system, so two passes over the same collateral do not score identically.", "note_thisPin": "This is rated by an AI system. Two passes over the same collateral scored within {{points}} percentage points of each other on the rater that read your entry.", "note_otherPin": "This is rated by an AI system, so two passes over the same collateral do not score identically. Your entry was read by {{ratedBy}} running {{ratedWith}}, and the published margin was measured on {{model}} running {{skills}}, so it does not describe this score.", "note_manyPins": "This is rated by an AI system, so two passes over the same collateral do not score identically. These scores were not all read by the same rater, so no one margin describes the board.", "note_notYetRated": "This is rated by an AI system. Two passes over the same collateral scored within {{points}} percentage points of each other on {{model}} running {{skills}}, and your scorecard will name what read your entry.", "link": "How the rating varies", "title": "The error margin", "lede": "An AI system reads your collateral, and reading is not arithmetic. Enter the same archive twice and the two scores will not be identical.", "pinned": "What holds that difference down is the pin: the deployment rates with one model running one version of the rating skills. Your scorecard records both, and two scores are comparable when they match.", "varies": "What a pin cannot fix is a criterion the evidence leaves genuinely close to the line between two bands. Two passes can read that one either way, and a criterion moving is enough to move the percentage.", "figure_unmeasured": "No figure is published yet. Nothing has been rated on this deployment, so there is no spread to report, and a margin carried over from another model or an earlier rubric would be the kind of unsourced number this benchmark exists to argue against. We will measure it by rating identical collateral repeatedly on the pinned rater, and publish it here naming that rater, the number of passes, and the spread.", "figure_notYetRated": "Rating identical collateral {{passes}} times on {{model}} running {{skills}} put the scores within {{points}} percentage points of each other. That is the margin for that rater and for no other: your scorecard names what read your entry, and a score read by something else is not covered by this number.", "appeal": "A re-grade is for a rater that misread your work, not for a score that landed slightly below the last one. The higher of the two stands either way, so an appeal can never cost you rank." }, "methodology": { "pdf": "Download this page as a PDF", "scrollCue": "Scroll, and watch an entry get rated", "cta": { "samples": "Pick a sample", "leaderboard": "See the leaderboard" }, "reveal": { "open": "How does that work?", "close": "Hide the detail", "label": "The detail" }, "beats": { "opening": { "title": "How scoring works", "body": "Your collateral goes in, and a score you can work out by hand comes out." }, "bundle": { "title": "It starts with a real block.", "bullets": { "spec": "The micro-architecture specification", "protocols": "The supporting protocol documents", "rtl": "The full RTL, with a filelist" } }, "register": { "title": "Register with any email.", "body": "It is verified by a one-time code, and then you give your name." }, "build": { "title": "Build it offline.", "body": "Your environment, your simulator, whatever tools you like." }, "enter": { "title": "Enter one archive.", "body": "No folder structure is prescribed: the raters discover what you sent." }, "raters": { "title": "Six raters read it.", "body": "Each dimension has its own rater, reading against its own criteria." }, "testplan": { "body": "Traceability to the spec, feature and register coverage, and planning for corners, errors and negatives." }, "drvMon": { "body": "Protocol correctness: the right side driven, at the right time, sampled on the right edge." }, "sbPred": { "body": "Golden-model fidelity, contract completeness, compare efficacy, and resilience through reset." }, "sva": { "body": "Protocol rules asserted correctly and completely, properly bound, reset-aware and never vacuous." }, "fcov": { "body": "Covergroups, bins, crosses and illegal bins: the model is rated, never a percentage." }, "testcases": { "body": "Feature and testplan-row coverage, self-checking efficacy, stimulus quality and traceability." }, "specification": { "title": "The specification is not rated.", "body": "It ships with the sample, so it is not your work." }, "readOnly": { "title": "They read. They never run.", "body": "No simulation, and not one edit to your collateral." }, "marks": { "title": "Every criterion is judged on four bands.", "body": "Each against its own written anchor, and scored out of 100 like every number here.", "bullets": { "poor": "0% Poor: absent, or too incomplete to serve its purpose", "partial": "33.3% Partial: present, but materially incomplete", "good": "66.7% Good: meets the criterion, with minor gaps", "excellent": "100% Excellent: complete, and usable as-is downstream" } }, "weighted": { "title": "Weight times score.", "body": "Every scored criterion adds its weight times its score to the top of a fraction." }, "mean": { "title": "Over the weights.", "body": "Its weight goes on the bottom, and the fraction is your score." }, "floor": { "title": "Measured from the floor.", "body": "The bottom band scores 0% and each band above it adds a third, to one decimal place." }, "zeroToHundred": { "title": "All Poor is 0%. All Excellent is 100%.", "body": "Not the 25% that counting Poor as one band in four would hand out for nothing." }, "noDimensionWeights": { "title": "No dimension weights.", "body": "A dimension pulls on your overall by the weight of what counted inside it." }, "notApplicable": { "title": "Not applicable is not zero.", "body": "A criterion that does not apply keeps its full rubric weight and leaves both sums." }, "pending": { "title": "Neither is pending.", "body": "A criterion only a simulator could settle leaves both sums as well." }, "gate": { "title": "A high average is not enough.", "body": "Structure alone does not catch a bug, so efficacy criteria spanning the whole pipeline are checked beside the average as a gate." }, "capped": { "title": "One cap blocks it.", "body": "If any one of them is capped, the entry cannot be marked ready to use, whatever the percentage says." }, "readyToUse": { "title": "Ready to use", "body": "Every efficacy criterion that applies to the block was scored clear of its cap." }, "blocked": { "title": "Blocked", "body": "At least one sits at or below its cap, and the verdict names which one." }, "unproven": { "title": "Unproven", "body": "Nothing was capped and nothing was shown either: a gap in the evidence, not a finding against your work." }, "coverageReports": { "title": "Coverage reports are never read.", "body": "A reported percentage is trivial to fabricate and impossible to verify." }, "provenance": { "title": "How you made it is never asked.", "body": "Our tools, a competitor's, or one engineer and a text editor all enter the same way." }, "pinned": { "title": "One pass, pinned.", "body": "Each entry is rated once under a pinned model and skills version, your scorecard names both, and two scores compare when their pins match." }, "appeal": { "title": "One re-grade on appeal.", "body": "The higher of the two scores stands, so appealing can never cost you rank." }, "reading": { "title": "Reading is not arithmetic.", "body": "Enter the same archive twice and the two scores will not be identical, though the pin holds them close." }, "spread": { "title": "So how far apart?", "body_unmeasured": "Not measured yet: nothing has been rated here, and we will not borrow a number from anywhere else.", "body_notYetRated": "Within {{points}} percentage points, across {{passes}} passes on {{model}} running {{skills}}." }, "bestCounts": { "title": "Your best entry counts.", "body": "Re-entering can raise your rank and never lower it." }, "podium": { "title": "Then you compete.", "body": "Your best entry per sample lands on the leaderboard, under your own name." }, "begin": { "title": "Ready when your collateral is.", "body": "Pick a sample, build it offline, and enter one archive." } }, "reveals": { "origin": "These raters come out of the tooling MooresLab built for its own verification work. Every entry is read under one pinned model running one pinned version of them.", "archive": { "format": "The format is read off the bytes, never off the file name.", "members": "Plain files and folders are taken as they are. A symbolic link becomes a copy of the file or folder it points to inside the archive, or is left out, and a device or any other special file is refused.", "duplicate": "The same archive cannot be entered twice.", "limits": "The size limits are printed above the file picker, so you know them before you build." }, "criterion": "A criterion states what is required, carries a weight, and carries four anchors, one for each score on the scale. Your scorecard names every criterion applied to your entry, the weight it carried, the score it was given, and the file and line that score rests on.", "workedExample": { "lead": "Made-up numbers, to show the arithmetic rather than anybody's rubric:", "row": "weight {{weight}}, score {{score}}%", "rowNotApplicable": "weight {{weight}}, not applicable", "score": "score = {{numerator}} / {{denominator}} = {{score}}%", "thirds": "33.3 and 66.7 are a third and two thirds, rounded. Your scorecard works with the thirds themselves, so a sum by hand can land a tenth of a point away from it.", "listing": "The same rule, as a verification engineer would write it:" }, "dimensionScore": "A dimension's score is that weighted mean over its own criteria, and your overall is the same mean over every criterion in the entry, so every number on your scorecard, each criterion's included, is out of 100. A dimension with nothing to rate is recorded not applicable with its reason and drops out, and nothing is rebalanced to cover for it.", "recordedApart": "Both leave the numerator and the denominator, so neither is a zero. They are recorded apart because \"this does not apply to your block\" and \"we did not check this\" are different sentences.", "efficacyExamples": { "lead": "Averaging six rubrics rewards structure. It does not, by itself, prove your environment would catch a bug. So the gate checks things such as:", "checker": "A checker that can disagree with a wrong DUT", "corner": "A corner the suite can actually enter", "field": "A field the monitor can observe", "rule": "A rule actually asserted, and not vacuously", "bin": "An error class with a bin", "planned": "Errors and corners planned and checked" }, "verdicts": { "lead": "The gate gives one of three answers, and the third is the one most often misread. A criterion that only a simulator could settle reports pending, so an efficacy criterion can end up neither capped nor shown." }, "fcovModel": "rate-fcov rates the coverage model in your source, the covergroups and bins and crosses you wrote, rather than a number a tool printed.", "board": "The board ranks entries, not contestants: each row is one contestant's best entry on one sample, and nothing averages anybody's entries across samples. Rows are ordered by score, then by the entry submitted first, then by the entry's id, so the board never reshuffles on a reload." }, "stage": { "bundle": { "spec": { "kicker": "Micro-architecture", "label": "Spec" }, "protocols": { "kicker": "Supporting", "label": "Protocols" }, "rtl": { "kicker": "Full source", "label": "RTL" } }, "oneTimeCode": "One-time code", "archive": { "kicker": "Your entry", "label": "One archive" }, "specification": "Specification", "notRated": "Not rated", "read": "Read", "run": "Run", "edit": "Edit", "weightTimesMark": "weight × score", "weight": "weight", "score": "score", "floor": "Poor = 0%", "notApplicable": "N/A", "pending": "Pending", "dimensionWeight": "dimension weight", "leavesBoth": "Leaves both sums", "average": "Average", "efficacyGate": "Efficacy gate", "capped": "Capped", "notRead": "Not read", "notAsked": "Not asked", "toolchain": "your toolchain", "scorecard": "Scorecard", "model": "Model", "skills": "Skills version", "pinned": "Pinned", "pinsMatch": "Pins match", "firstPass": "First pass", "regrade": "Re-grade", "stands": "Stands", "evidence": "Evidence", "passOne": "Pass 1", "passTwo": "Pass 2", "best": "Best", "bands": { "poor": "Poor", "partial": "Partial", "good": "Good", "excellent": "Excellent" }, "weightIs": "weight {{weight}}", "weightAndScore": "weight {{weight}}, score {{score}}%", "allAt": "every criterion {{band}}", "high": "High", "yourSource": "Your source", "entry": "Entry {{number}}" }, "rubricSource": { "title": "Where the criteria are published", "body_unpublished": "Your scorecard names every criterion that was applied to your entry, the weight it carried, the score it was given, and the file and line that score rests on, so nothing is scored against a standard you cannot read back afterwards. The rating skills that define those criteria are being prepared for publication under a licence that requires citation.", "body_published": "All six raters are published, so the whole standard is readable before you build against it. Your scorecard then names every criterion that was applied to your entry, the weight it carried, the score it was given, and the file and line that score rests on.", "pinned": "It also records the version of those skills that rated you, so the criteria you were measured against are the ones that version carried.", "link": "Read the rating skills" }, "fairness": { "published": { "title_published": "Published rubrics", "title_unpublished": "No hidden standard", "body_published": "Every criterion, its weight and its four anchors are public before you enter.", "body_unpublished": "The criteria applied to your entry are on your scorecard, and publishing the rating skills that define them is next." } } }, "secretExpiry": { "title_one": "A secret needs rotating", "title_other": "{{count}} secrets need rotating", "llmAccount": "An LLM account", "githubToken": "GitHub error reporting token", "slackWebhook": "Slack webhook for #benchmark-admin", "expired": "{{name}} expired on {{date}}. Whatever it signs in to is already refusing it.", "expiring_one": "{{name}} expires in {{count}} day, on {{date}}.", "expiring_other": "{{name}} expires in {{count}} days, on {{date}}.", "rotationOverdue": "{{name}} was due to be rotated on {{date}}. It still posts; save a new one to keep to the schedule.", "rotationDue_one": "{{name}} is due to be rotated in {{count}} day, on {{date}}.", "rotationDue_other": "{{name}} is due to be rotated in {{count}} days, on {{date}}.", "rotate": "Rotate", "window_one": "Operators are warned {{count}} day before a recorded expiry. Only operators see this.", "window_other": "Operators are warned {{count}} days before a recorded expiry. Only operators see this.", "jiraToken": "Jira API token for known-bug scoring" }, "files": { "loading": "Loading the entry's files", "failed": "We could not load this entry's files", "gone": { "title": "These files are no longer kept", "body": "An entry's archive is deleted 90 days after it is entered. The entry and its rating stay; the files it was rated on do not." }, "notFound": { "title": "These files are not available", "body": "There is no entry here whose files this account can read." }, "empty": { "title": "This archive holds no files", "body": "The archive that was entered contains folders and nothing in them." }, "tree": { "filter": "Filter any part of the path", "expand": "Expand {{name}}", "collapse": "Collapse {{name}}" }, "start": { "title": "Start with one of these", "lede": "The rest of the entry is in the tree. Drag a tab to the edge of this pane to read two files side by side.", "opensIn": { "monaco": "{{size}}, opens in the editor.", "waveform": "{{size}}, opens in the waveform viewer.", "document": "{{size}}, opens as a document.", "image": "{{size}}, opens as an image.", "csv": "{{size}}, opens as a table.", "rendered": "{{size}}, opens rendered.", "download": "{{size}}, download only.", "tooLarge": "{{size}}, too large to open here." } }, "viewer": { "mode": "How to read this file", "preview": "Preview", "raw": "Raw", "both": "Both", "download": "Download", "downloadIt": "Download it", "unreadable": "This file could not be read.", "cannotShow": "This file is {{size}} and is not something the browser can show.", "tooLarge": "This file is {{size}}, which is too large to open here. Files up to 64 MB open in the browser." }, "csv": { "label": "{{path}}, as a table", "noColumns": "This CSV has no columns. Switch to Raw to see what it holds.", "capped": "First {{shown}} of {{total}} rows. Raw shows all of it.", "expandHint": "Pick a row to read its full cells.", "unnamedColumn": "Column {{number}}" }, "pdf": { "loading": "Loading the document", "unreadable": "This PDF could not be read.", "back": "Back", "next": "Next", "position": "{{page}} / {{count}}", "fullscreen": "Read it fullscreen", "fullscreenHint": "Every page, full height", "pages_one": "{{count}} page, page keys scroll", "pages_other": "{{count}} pages, page keys scroll" }, "waveform": { "loading": "Loading the waveform", "unreadable": "This waveform could not be read.", "empty": "This waveform is empty.", "emptyFile": "This VCD file has no content." } }, "reportForm": { "action": "Report", "hint": "Tell the people who run the benchmark something is wrong with this.", "titles": { "scorecard": "Report this scorecard", "comment": "Report this comment", "profile": "Report this page" }, "lede": "Say what is wrong. Somebody who runs the benchmark reads every report and decides what happens, and the person you report is not told who you are.", "reason": "What is wrong", "reasonHint": "At least {{min}} characters.", "confirm": "Send report", "sent": "Thank you. Somebody who runs the benchmark will review it.", "failed": "Your report did not go through. Try again.", "signedOut": { "title": "Sign in to report this", "body": "Reports come from accounts, so that one person cannot report the same thing over and over.", "action": "Sign in" }, "commentHint": "Tell the people who run the benchmark something is wrong with this comment.", "profileHint": "Tell the people who run the benchmark something on this page breaks the rules." }, "entrantFeedback": { "standings": { "awaiting": "Awaiting a reply", "open": "Open", "closed": "Closed" }, "fromScorecard": { "action": "Send feedback", "hint": "Ask about this entry's rating, or tell us something is wrong. A reply arrives by email." }, "fromDimension": { "action": "Send feedback", "hint": "Tell us about how {{dimension}} was rated. The form opens with this entry and this dimension chosen." }, "categories": { "general": "General", "testplan": "Test plan", "drv-mon": "Drivers and monitors", "sb-pred": "Scoreboard and predictor", "sva": "Assertions", "fcov": "Functional coverage", "testcases": "Test suite", "website": "The website" }, "new": { "title": "Send feedback", "about": "Ask a question, raise a concern about a rating, or tell us what could be better. The people who run the benchmark read every message.", "byEmail": "There is no list of the feedback you send. When somebody replies, we email you a link to the conversation, and that email is how you come back to it.", "category": "What it is about", "categoryHint": "A dimension of the rating, the rating as a whole, or the website.", "entry": "About an entry", "noEntry": "Not about one entry", "thisEntry": "The entry you came from", "message": "Your message", "messageHint": "At least {{min}} characters, as plain text.", "send": "Send feedback" }, "thread": { "untitled": "Feedback", "aboutTitle": "this conversation", "aboutAuthor": "Your feedback and every reply to it. When somebody replies, we email you a link back here; there is no list of your feedback anywhere else on the site.", "aboutOperator": "One entrant's feedback and every reply. Your reply emails them a link to this page and nothing of what you wrote, and closing the conversation stops either side from writing until it is reopened.", "failed": "We could not load this conversation", "notFound": "No conversation of yours lives at this address.", "sentTitle": "Sent. Thank you.", "sentBody": "When somebody replies, we email {{email}} a link back to this page. Keep that email: the site keeps no list of your feedback.", "category": "Category:", "startedBy": "Started by", "about": "About", "unnamed": "An entrant without a name", "fromYou": "You", "fromTeam": "The MooresLab Benchmark team", "fromOperator": "{{name}}, operator", "addLabel": "Add to this conversation", "addHint": "The people who run the benchmark are told. You are emailed when they reply.", "addAction": "Send", "replyLabel": "Reply", "replyHint": "The entrant is emailed a link to this conversation, and nothing of what you write.", "replyAction": "Send reply", "closedAuthor": "This conversation is closed. If there is more to say, send new feedback.", "closedOperator": "This conversation is closed. Reopen it to reply.", "close": "Close conversation", "reopen": "Reopen conversation", "byEmail": "By email" }, "attachments": { "attach": "Attach files", "limits": "Up to {{count}} files, {{each}} each and {{together}} together. Every image is checked before it is kept.", "chosen": "Files to send", "remove": "Remove {{name}}", "sent": "Attached files", "open": "Open {{name}}", "refused": { "tooMany": "A message carries at most {{count}} files. Zip the rest, or send them in another message.", "tooLarge": "{{name}} is larger than {{each}}. Zip it, or send a smaller file.", "tooLargeTogether": "{{name}} would take these files past {{together}} together. Send it in another message.", "empty": "{{name}} is empty. Choose the file you meant to send." } }, "writeSomething": "Write something first.", "failedToSend": "That did not send. Try again in a moment." }, "refusals": { "account_address_disposable": "Addresses at {{domain}} are throwaway mailboxes this site does not accept. Use an address you keep.", "account_already_set_up": "This account is already set up. Change your name in account settings.", "account_last_operator": "This deployment would have no operator left. Appoint another operator first.", "account_operator_role_own": "You cannot remove your own operator role. Ask another operator to do it.", "account_signup_unreadable": "That sign-up could not be read whole. Try again from the sign-up page.", "account_suspend_own": "You cannot suspend your own account. Ask another operator to do it.", "account_suspension_reason_missing": "Say why this account is suspended. The entrant is shown the reason.", "account_suspension_reason_too_long_one": "A reason is at most {{count, number}} character.", "account_suspension_reason_too_long_other": "A reason is at most {{count, number}} characters.", "account_temporary_password_own": "You cannot reset your own password here. Change it in your security settings.", "alert_address_malformed": "That is not an email address. Type the whole address, like alerts@example.com.", "alert_address_taken": "That address is already on the list.", "alert_bugs_found_not_emailed": "Nobody is emailed about bug counts; they post to Slack alone.", "alert_deletions_not_emailed": "Nobody is emailed about deleted entries; they post to Slack alone.", "alert_entries_not_emailed": "Nobody is emailed about new entries; they post to Slack alone.", "alert_error_issues_not_emailed": "Nobody is emailed about filed issues; they post to Slack alone.", "alert_kinds_empty": "Choose at least one kind, or turn this admin off instead.", "alert_name_not_one_line": "A name is one line of plain text.", "alert_name_too_long_one": "A name is at most {{count, number}} character.", "alert_name_too_long_other": "A name is at most {{count, number}} characters.", "alert_ratings_not_emailed": "Nobody is emailed about ratings; they post to Slack alone.", "alert_sign_ups_not_emailed": "Nobody is emailed about sign-ups; they post to Slack alone.", "appeal_account_suspended": "This account is suspended, so it cannot appeal. Your profile says why.", "appeal_already_running": "A rating pass is already running on this entry. Wait for it to finish, then appeal if you still disagree.", "appeal_argument_too_long_one": "Keep the appeal under {{count, number}} character. Name what was misread and where it lives.", "appeal_argument_too_long_other": "Keep the appeal under {{count, number}} characters. Name what was misread and where it lives.", "appeal_argument_too_short_one": "Say what the raters got wrong, in at least {{count, number}} character. A re-grade nobody explained is one nobody can learn from.", "appeal_argument_too_short_other": "Say what the raters got wrong, in at least {{count, number}} characters. A re-grade nobody explained is one nobody can learn from.", "appeal_no_rater": "This deployment cannot run a re-grade right now. That is ours to fix and nothing to do with your collateral, so please come back shortly and appeal then.", "appeal_nothing_rated": "Nothing has rated this entry yet, so there is nothing to appeal. If a pass failed on our side an operator will run it again, and your appeal stays yours either way.", "appeal_spent": "You have already had the one re-grade an appeal buys, and the higher of the two scores stands. If that pass failed on our side, ask us to run it again: a re-run costs you nothing.", "appeal_taken_down": "This entry was taken down, so it cannot be rated again. Its scorecard says why.", "archive_backslash_name": "That archive writes its paths with backslashes, which an old DOS-era zip tool does. Re-zip it with Explorer, 7-Zip or zip, and send it again.", "archive_bomb": "Unpacking that archive would produce more than {{limit, megabytes}}. Send the collateral, not everything the run produced.", "archive_contradiction": "That archive could not be read through to the end (two of its members cannot both exist). Rebuild it and send it again.", "archive_corrupt": "That archive could not be read through to the end (a member's data could not be decoded). Rebuild it and send it again.", "archive_counts_differ": "That archive could not be read through to the end (its end record counts the members differently from the directory). Rebuild it and send it again.", "archive_directory_disagrees": "That archive could not be read through to the end (its central directory describes a member differently from the bytes it holds). Rebuild it and send it again.", "archive_empty": "Attach the archive holding your collateral.", "archive_encrypted": "That archive is password-protected. Send it without a password.", "archive_encrypted_aes": "That archive is password-protected (AES). Send it without a password.", "archive_file_and_folder": "That archive could not be read through to the end (it lists a file and a folder under one name). Rebuild it and send it again.", "archive_folder_with_content": "That archive could not be read through to the end (it lists a folder that carries a file's bytes). Rebuild it and send it again.", "archive_header_disagrees": "That archive could not be read through to the end (a member's header does not name what the directory lists there). Rebuild it and send it again.", "archive_malformed": "That archive could not be read through to the end (a header in it is not shaped the way a zip's is). Rebuild it and send it again.", "archive_method": "That archive uses a compression method this benchmark does not read (method {{method}}). Re-zip it with the standard Deflate method.", "archive_method_family": "That archive uses {{family}} compression, which this benchmark does not read. Re-zip it with the standard Deflate method, which every zip tool offers (7-Zip calls it Deflate).", "archive_misplaced_directory": "That archive could not be read through to the end (its end record places the central directory somewhere other than where it is). Rebuild it and send it again.", "archive_more_members_than_held": "That archive could not be read through to the end (its central directory describes more members than the archive holds). Rebuild it and send it again.", "archive_name_not_unicode": "That archive holds a file name we cannot read safely: its zip tool wrote it in a character set other than Unicode. Zip it again with 7-Zip, which writes Unicode names, and send it again.", "archive_name_too_long_one": "That archive holds a file path longer than {{count, number}} byte once written in Unicode, where an accented letter takes two and a Chinese or Japanese one three. Shorten the folder names and send it again.", "archive_name_too_long_other": "That archive holds a file path longer than {{count, number}} bytes once written in Unicode, where an accented letter takes two and a Chinese or Japanese one three. Shorten the folder names and send it again.", "archive_names_twice": "That archive could not be read through to the end (its central directory names one member twice). Rebuild it and send it again.", "archive_no_members": "That archive is empty. Put your collateral in it and send it again.", "archive_not_a_file": "That archive holds {{name}}, which is not a file or a folder. Send plain files: no symbolic links, device nodes, fifos or sockets.", "archive_not_a_record": "That archive could not be read through to the end (its central directory holds something that is not a record). Rebuild it and send it again.", "archive_not_a_zip": "That file is not a zip archive. Send one .zip with everything in it.", "archive_second_end_record": "That archive could not be read through to the end (it hides a second end record where a reader would find it first). Rebuild it and send it again.", "archive_sizes_differ": "That archive could not be read through to the end (its end record sizes the directory differently from what it holds). Rebuild it and send it again.", "archive_stalled_one": "That upload stalled: nothing arrived for {{count, number}} second, so it was stopped. Check your connection and send it again.", "archive_stalled_other": "That upload stalled: nothing arrived for {{count, number}} seconds, so it was stopped. Check your connection and send it again.", "archive_stored_without_marker": "That archive is laid out in a way this benchmark cannot read yet. Nothing is wrong with your collateral, and we have been told; re-zipping it with 7-Zip or Info-ZIP zip usually goes through in the meantime.", "archive_too_large": "That archive is larger than {{limit, megabytes}}. An entry is source, not simulation output: leave waveforms and build products out.", "archive_too_many_entries_one": "That archive holds more than {{count, number}} file.", "archive_too_many_entries_other": "That archive holds more than {{count, number}} files.", "archive_too_many_folders_one": "That archive's members imply more than {{count, number}} folder between them. Flatten it and send it again.", "archive_too_many_folders_other": "That archive's members imply more than {{count, number}} folders between them. Flatten it and send it again.", "archive_too_many_images_one": "That archive holds more than {{count, number}} image. Every image is checked before an upload is kept, so send the diagrams and screenshots the collateral needs and leave the rest out.", "archive_too_many_images_other": "That archive holds more than {{count, number}} images. Every image is checked before an upload is kept, so send the diagrams and screenshots the collateral needs and leave the rest out.", "archive_trailing_bytes": "That archive could not be read through to the end (it is followed by bytes that are not part of the archive). Rebuild it and send it again.", "archive_truncated": "That archive could not be read through to the end (it ends before its central directory does). Rebuild it and send it again.", "archive_unexpected_header": "That archive could not be read through to the end (a header in it is not where a zip puts one). Rebuild it and send it again.", "archive_unsafe_name": "That archive holds a file whose path leaves the archive ({{name}}). Send relative paths only.", "budget_ceiling_empty": "Name a ceiling of at least one token. A ceiling of none would accept entries and rate none of them.", "client_error_message_missing": "A report needs a message.", "comment_account_suspended": "This account is suspended, so it cannot comment or react. Your profile says why.", "comment_control_characters": "A comment cannot contain control characters. Retype it rather than pasting it, and try again.", "comment_email_unverified": "Verify your email address before commenting. The banner at the top of the page sends the code again.", "comment_empty": "Write something before posting.", "comment_hourly_allowance_one": "Too many requests. Try again in {{count, number}} second.", "comment_hourly_allowance_other": "Too many requests. Try again in {{count, number}} seconds.", "comment_name_missing": "Add your first and last name before commenting. They are what your comments are signed with.", "comment_terms_outstanding": "Read and accept the current terms of service and privacy policy before commenting. They are linked in the footer of every page.", "comment_too_long_one": "Keep a comment under {{count, number}} character.", "comment_too_long_other": "Keep a comment under {{count, number}} characters.", "delivery_unblock_no_key": "This deployment holds no Brevo key, so it cannot ask Brevo to unblock an address. Unblock it in Brevo's transactional blocklist instead.", "delivery_unblock_refused": "Brevo did not unblock this address, so it is still blocked. Try again in a moment, or unblock it in Brevo's transactional blocklist.", "entries_per_day": "You have entered five times in the last day, which is the allowance. Entries that were never rated do not count, whether you deleted them or not; try again tomorrow.", "entry_account_suspended": "This account is suspended, so it cannot enter. Your profile says why.", "entry_already_entered": "You entered this exact archive on {{at, date}}, as entry {{id}}. Change the collateral, or appeal that entry instead.", "entry_archive_several": "Send one archive with everything in it, not several.", "entry_delete_closed": "This entry is already deleted or taken down.", "entry_delete_running": "A rater is reading this entry right now, so it cannot be deleted until that pass finishes. Try again once its scorecard shows the result.", "entry_delete_under_review": "This entry is held for review, and an operator decides what happens to it. Send feedback about it from its scorecard if you want it deleted.", "entry_file_path_invalid": "That is not a path inside an archive.", "entry_file_path_missing": "Name the file to open.", "entry_file_too_large_to_view": "This file is too large to open here.", "entry_file_unsupported": "This file is stored in a way the viewer cannot open: it is encrypted, or compressed with something other than deflate.", "entry_files_gone": "This entry's files are no longer kept. An archive is deleted 90 days after it is entered; the entry and its rating stay.", "entry_form_unreadable": "That upload could not be read. Send a multipart form with the sample slug first and the archive last.", "entry_identical_archive": "An identical archive has already been entered, and each archive can be entered only once. Change the collateral to enter it.", "entry_name_missing": "Add your first and last name before entering. They are what the leaderboard shows.", "entry_no_rater": "This deployment is not ready to rate entries yet. That is ours to fix and nothing to do with your collateral, so please come back shortly and enter it then.", "entry_notes_too_long_one": "Keep the notes to the raters under {{count, number}} character. They are context, not an argument for a score.", "entry_notes_too_long_other": "Keep the notes to the raters under {{count, number}} characters. They are context, not an argument for a score.", "entry_sample_missing": "Name the sample this entry is against, in a field before the archive.", "entry_state_unknown": "An entry's state is one of: {{choices}}.", "entry_terms_outstanding": "Read and accept the current terms of service and privacy policy before entering. They are linked in the footer of every page.", "entry_upload_secret_missing": "Send the secret start_entry gave you in the {{header}} header.", "entry_upload_unusable": "This upload URL is unknown, already used, or expired. Ask for a new one with start_entry, then upload to it within five minutes.", "entry_upload_urls_waiting_one": "You already hold {{count, number}} upload URL nobody has used. Upload to it, or wait five minutes for it to expire.", "entry_upload_urls_waiting_other": "You already hold {{count, number}} upload URLs nobody has used. Upload to one of them, or wait five minutes for them to expire.", "feedback_account_suspended": "This account is suspended, so it cannot send feedback. Your profile says why.", "feedback_attachment_empty": "That attachment is empty. Choose the file you meant to send.", "feedback_attachment_name_control_characters": "An attachment's name cannot hold control characters. Rename the file and send it again.", "feedback_attachment_too_large": "An attachment is at most {{limit, megabytes}}, and a message's attachments are at most {{combined_limit, megabytes}} together. Zip a large file, or send it in a message of its own.", "feedback_attachments_excess_one": "A message carries at most {{count, number}} attachment. Zip the rest, or send them in a message of their own.", "feedback_attachments_excess_other": "A message carries at most {{count, number}} attachments. Zip the rest, or send them in a message of their own.", "feedback_category_unknown": "Choose one of the listed categories.", "feedback_closed_to_author": "This conversation is closed, so it takes no more messages. Send new feedback from the account menu if there is more to say.", "feedback_closed_to_operator": "This conversation is closed. Reopen it before replying.", "feedback_daily_allowance_one": "Too many requests. Try again in {{count, number}} second.", "feedback_daily_allowance_other": "Too many requests. Try again in {{count, number}} seconds.", "feedback_email_unverified": "Verify your email address before sending feedback. A reply reaches you by email, and the banner at the top of the page sends the link again.", "feedback_entry_unknown": "That entry is not one this site knows.", "feedback_form_unreadable": "That message could not be read. Send its words first and its attachments last.", "feedback_message_control_characters": "A message cannot contain control characters. Retype it rather than pasting it, and try again.", "feedback_message_empty": "Write something before sending.", "feedback_message_too_long_one": "A message is at most {{count, number}} character.", "feedback_message_too_long_other": "A message is at most {{count, number}} characters.", "feedback_message_too_short_one": "Say what is on your mind in at least {{count, number}} character.", "feedback_message_too_short_other": "Say what is on your mind in at least {{count, number}} characters.", "feedback_name_missing": "Add your first and last name before sending feedback.", "feedback_status_unknown": "A conversation's status is one of: open, closed, all.", "feedback_subject_not_one_line": "A subject is one line of plain text.", "feedback_subject_too_long_one": "A subject is at most {{count, number}} character.", "feedback_subject_too_long_other": "A subject is at most {{count, number}} characters.", "feedback_terms_outstanding": "Read and accept the current terms of service and privacy policy before sending feedback. They are linked in the footer of every page.", "feedback_words_after_attachments": "Send the message's words before its attachments, not after them.", "file_name_missing": "Give the file a name, or upload one that has one.", "file_name_too_long_one": "A file name is at most {{count, number}} character.", "file_name_too_long_other": "A file name is at most {{count, number}} characters.", "form_category_not_text": "That category is not text.", "form_category_too_long": "That category is far longer than one can be.", "form_entry_not_text": "That entry is not text.", "form_entry_too_long": "That entry is far longer than one can be.", "form_file_name_not_text": "That file name is not text.", "form_file_name_too_long": "That file name is far longer than one can be.", "form_kind_not_text": "That kind is not text.", "form_kind_too_long": "That kind is far longer than one can be.", "form_message_not_text": "That message is not text.", "form_message_too_long": "That message is far longer than one can be.", "form_notes_not_text": "That notes is not text.", "form_notes_too_long": "That notes is far longer than one can be.", "form_page_count_not_text": "That page count is not text.", "form_page_count_too_long": "That page count is far longer than one can be.", "form_position_not_text": "That position is not text.", "form_position_too_long": "That position is far longer than one can be.", "form_sample_slug_not_text": "That sample slug is not text.", "form_sample_slug_too_long": "That sample slug is far longer than one can be.", "form_source_not_text": "That source is not text.", "form_source_too_long": "That source is far longer than one can be.", "form_subject_not_text": "That subject is not text.", "form_subject_too_long": "That subject is far longer than one can be.", "form_unreadable": "That upload could not be read.", "form_version_not_text": "That version is not text.", "form_version_too_long": "That version is far longer than one can be.", "github_expiry_passed": "An expiration date has to be in the future.", "github_repository_malformed": "Name the repository as owner/name, such as mooreslabaiv1/benchmark.", "github_test_not_configured": "Save a token first: there is nothing to file with yet.", "github_test_refused": "{{detail}}", "github_test_unavailable": "{{detail}}", "github_token_empty": "Paste the token, or leave the field out to keep the saved one.", "github_token_malformed": "That is not a GitHub token. Copy it again from the token page.", "github_token_required": "Paste a token: none is saved yet, so there is nothing to keep.", "image_refused": "This upload holds an image we could not accept. Images are checked automatically: each must be a PNG, JPEG, WebP, GIF, BMP or single-page TIFF of at most {{limit, megabytes}} and {{megapixels, number}} megapixels, and no sexually explicit image is accepted. Remove it and send the upload again.", "image_screen_unavailable": "Images in uploads are checked automatically, and the check is not available right now. That is ours to fix and nothing to do with your upload, so please send it again shortly.", "internal": "Something went wrong on our side.", "jira_email_malformed": "Give the address of the Atlassian account the token belongs to.", "jira_expiry_passed": "The token's expiry has to be in the future: Atlassian shows it beside the token.", "jira_project_key_malformed": "A project key is a capital letter and then one to nine capitals, digits or underscores, such as SAMPLES.", "jira_site_malformed": "That is not a Jira Cloud site. Give its address as https://.atlassian.net, with nothing after it.", "jira_token_malformed": "That is not an Atlassian API token. Copy it whole from the account's API tokens page.", "jira_token_required": "Paste the API token: none is saved yet, so there is nothing to keep.", "leaderboard_too_many_samples_one": "Ask about at most {{count, number}} sample at a time.", "leaderboard_too_many_samples_other": "Ask about at most {{count, number}} samples at a time.", "legal_document_changed": "This document has changed since the page was opened. Reload and read the current version before accepting it.", "legal_document_unknown": "That is not a document this deployment publishes.", "legal_documents_excess": "That is more documents than this deployment publishes.", "legal_documents_missing": "Name the documents you are accepting.", "llm_expiry_passed": "That expiry has already passed. Give the date the secret stops working, or leave it blank if it does not expire.", "llm_label_empty": "Give the account a label, so the next operator knows whose it is.", "llm_label_taken": "An account with that label is already registered. Labels are compared without regard to capitalization.", "llm_label_too_long_one": "Keep the label under {{count, number}} character.", "llm_label_too_long_other": "Keep the label under {{count, number}} characters.", "llm_provider_unknown": "Name how the account authenticates: {{options}}.", "llm_secret_empty": "Paste the token or key itself.", "llm_secret_too_long": "That is far longer than a token or key is.", "llm_status_unknown": "A status is one of: {{options}}.", "not_found": "Not found.", "password_change_required": "Choose a new password before you continue.", "pin_account_disabled": "That account is turned off, so a pass could not sign in with it. Turn it back on, or pin another.", "pin_account_exhausted": "That account is at its usage limit, so a pass could not sign in with it. Pin it once the window has reset, or pin another.", "pin_account_expired": "That account's secret has passed the expiry recorded for it, so a pass could not sign in with it. Paste in a new secret with its new date, or pin another.", "pin_account_invalid": "The vendor rejected that account's secret, so a pass could not sign in with it. Replace the secret, or pin another.", "pin_account_unknown": "That account is not in the registry. Register it first, then pin it.", "pin_model_empty": "Name the model id this deployment rates under.", "pin_model_too_long_one": "Keep the model id under {{count, number}} character.", "pin_model_too_long_other": "Keep the model id under {{count, number}} characters.", "pin_skills_version_unknown": "That version of the rating skills is not stored here. Upload it first, then pin it.", "profile_bio_too_long_one": "A bio is at most {{count, number}} character.", "profile_bio_too_long_other": "A bio is at most {{count, number}} characters.", "profile_country_unknown": "Choose your country from the list. It is stored as its two-letter ISO code.", "profile_first_name_missing": "Enter your first name. It is how the leaderboard names you.", "profile_first_name_too_long_one": "A first name is at most {{count, number}} character.", "profile_first_name_too_long_other": "A first name is at most {{count, number}} characters.", "profile_last_name_missing": "Enter your last name. It is how the leaderboard names you.", "profile_last_name_too_long_one": "A last name is at most {{count, number}} character.", "profile_last_name_too_long_other": "A last name is at most {{count, number}} characters.", "profile_link_has_credentials": "{{link}} carries a user name or password. Remove it before listing the link.", "profile_link_listed_twice": "{{link}} is listed twice. List each link once.", "profile_link_not_https": "{{link}} is not a link we can list. Use a full https:// address, such as https://github.com/you.", "profile_link_too_long_one": "A link is at most {{count, number}} character.", "profile_link_too_long_other": "A link is at most {{count, number}} characters.", "profile_links_excess_one": "A profile lists at most {{count, number}} link.", "profile_links_excess_other": "A profile lists at most {{count, number}} links.", "profile_removal_nothing_chosen": "Choose what to remove: the bio, the links, the picture, or more than one.", "profile_removal_nothing_present": "Nothing you chose is on this page any more, so there is nothing to remove.", "profile_removal_own_page": "This is your own page. Change it in your account settings.", "profile_removal_reason_missing": "Say why this is being removed. The contestant is shown the reason.", "profile_removal_reason_too_long_one": "A reason is at most {{count, number}} character.", "profile_removal_reason_too_long_other": "A reason is at most {{count, number}} characters.", "range_unknown": "Name a range of 7d, 30d or all.", "report_account_suspended": "This account is suspended, so it cannot report anything. Your profile says why.", "report_already_filed": "You already reported this, and an operator has not reviewed it yet. One report is enough: every one is read.", "report_already_reviewed": "Somebody already reviewed this report. The list shows what they decided.", "report_dismiss_entrant_suspended": "This entry belongs to a suspended account, so dismissing the flag would leave it held with nothing to review. Reinstate the account first, or take the entry down.", "report_dismiss_not_pinned": "Nothing is pinned to rate with, so dismissing the flag cannot queue this entry. Pin a rater first.", "report_email_unverified": "Verify your email address before reporting anything. The banner at the top of the page sends the link again.", "report_kind_unknown": "A report's kind is one of: entry, scorecard, comment, profile.", "report_name_missing": "Add your first and last name before reporting anything.", "report_note_too_long_one": "A note is at most {{count, number}} character.", "report_note_too_long_other": "A note is at most {{count, number}} characters.", "report_own_words": "That is yours. Change or delete your own words instead of reporting them.", "report_page_parts_gone": "Nothing you chose is on this page any more. Dismiss the report instead.", "report_reason_too_long_one": "A reason is at most {{count, number}} character.", "report_reason_too_long_other": "A reason is at most {{count, number}} characters.", "report_reason_too_short_one": "Say what is wrong in at least {{count, number}} character. An operator reads every report, and the reason is what they act on.", "report_reason_too_short_other": "Say what is wrong in at least {{count, number}} characters. An operator reads every report, and the reason is what they act on.", "report_rescan_entrant_suspended": "This entry belongs to a suspended account, so it would stay held with nothing to review. Reinstate the account first, or take the entry down.", "report_rescan_not_held": "This entry is not held, so there is nothing to scan again. Dismiss the report or take the entry down instead.", "report_rescan_not_pinned": "Nothing is pinned to rate with, so the pre-scan cannot run. Pin a rater first.", "report_rescan_not_prescan": "Only the pre-scan's report on an entry can send it back to the pre-scan.", "report_status_unknown": "A report's status is one of: open, dismissed, actioned, rescanned, all.", "report_takedown_reason_missing": "Say why this entry is being taken down. The entrant is shown the reason.", "report_target_unknown": "A scorecard, a comment or a contestant's page can be reported. Say which this is.", "report_terms_outstanding": "Read and accept the current terms of service and privacy policy before reporting anything. They are linked in the footer of every page.", "reports_per_day_one": "Too many requests. Try again in {{count, number}} second.", "reports_per_day_other": "Too many requests. Try again in {{count, number}} seconds.", "rerun_account_suspended": "This entry belongs to a suspended account, so no pass runs for it. Reinstate the account first.", "rerun_already_running": "A rating pass is already queued or running on this entry. Wait for it to finish, then run it again if it still needs one.", "rerun_cancelled": "Its entrant cancelled this entry and its archive is deleted, so nothing rates it again.", "rerun_held": "The pre-scan flagged this entry and it is held for review. Dismiss or take down its report in Reported content; dismissing it queues the rating.", "rerun_no_rater": "This deployment has pinned no rater, so a re-run has nothing to run under. Pin one on the raters screen and try again.", "rerun_reason_missing": "Say why this entry is being run again. A re-run costs a full pass, and the log has to answer what it was for.", "rerun_reason_too_long_one": "A reason is at most {{count, number}} character.", "rerun_reason_too_long_other": "A reason is at most {{count, number}} characters.", "rerun_taken_down": "This entry was taken down, so nothing rates it again. A take-down is not undone by a re-run.", "roster_role_unknown": "A role is one of: {{names}}.", "roster_status_unknown": "A status is one of: {{names}}.", "sample_difficulty_unknown": "Difficulty must be one of {{choices}}.", "sample_file_empty": "That file is empty. Upload the one an entrant needs.", "sample_file_kind_missing": "Say what kind of file this is, in a field before the file.", "sample_file_kind_unknown": "File kind must be one of {{choices}}.", "sample_file_missing": "Attach the file to upload.", "sample_file_name_taken": "That sample already lists a file with this name.", "sample_file_several": "Send one file per request, not several.", "sample_file_too_large": "That file is larger than {{limit, megabytes}}, which is the most one sample file may be.", "sample_form_unreadable": "That upload could not be read. Send a multipart form with the file's kind first and the file last.", "sample_has_entries": "This sample has entries against it. Unpublish it instead, which hides it from the library and leaves the results intact.", "sample_jira_ip_too_long_one": "A Jira IP is at most {{count, number}} character.", "sample_jira_ip_too_long_other": "A Jira IP is at most {{count, number}} characters.", "sample_name_blank": "A name cannot be blank.", "sample_page_count_negative": "The page count cannot be negative.", "sample_page_count_not_whole": "The page count must be a whole number.", "sample_position_negative": "The position cannot be negative.", "sample_position_not_whole": "The position must be a whole number.", "sample_slug_shape": "A slug is lowercase letters, digits and single hyphens, e.g. axi4lite-to-apb3-bridge.", "sample_slug_taken": "Another sample already uses that slug.", "skills_archive_bomb": "Unpacking that archive would produce more than {{limit, megabytes}}, which is far more than the rating skills are.", "skills_archive_empty": "Attach the archive holding the rating skills.", "skills_archive_never_uploaded": "Nothing was ever uploaded against that version.", "skills_archive_several": "Send one archive holding the skills, not several.", "skills_archive_too_large": "That archive is larger than {{limit, megabytes}}. The rating skills are prompts: send the skills, not a checkout of the repository they live in.", "skills_current_raced": "Another operator made a different version current while this was saving. Reload, and choose again.", "skills_form_unreadable": "That upload could not be read. Send a multipart form with the version first and the archive last.", "skills_notes_too_long_one": "Keep the notes under {{count, number}} character.", "skills_notes_too_long_other": "Keep the notes under {{count, number}} characters.", "skills_nothing_uploaded": "Nothing was ever uploaded against that version, so no pass could run it. Remove it and upload the archive again.", "skills_pinned": "This deployment is pinned to that version. Pin another one first, then remove it.", "skills_preflight_not_failed": "This version's preflight has not failed, so there is nothing to run again: a pending one runs before the next pass, and an answered one is the version's answer.", "skills_source_too_long_one": "Keep the source under {{count, number}} character.", "skills_source_too_long_other": "Keep the source under {{count, number}} characters.", "skills_version_blank": "Name the version these skills are. Every rating records it, so it has to say which skills read the entry.", "skills_version_frozen_one": "{{count, number}} rating pass ran version {{version}}, so it cannot be changed or removed. Upload a correction as a new version.", "skills_version_frozen_other": "{{count, number}} rating passes ran version {{version}}, so it cannot be changed or removed. Upload a correction as a new version.", "skills_version_missing": "Name the version these skills are, in a field before the archive.", "skills_version_taken": "A version with that name is already stored, and a stored version never changes. Upload a correction under a new version.", "skills_version_too_long_one": "Keep the version under {{count, number}} character.", "skills_version_too_long_other": "Keep the version under {{count, number}} characters.", "slack_rotation_passed": "A rotation date has to be in the future.", "slack_test_not_configured": "Save a webhook first: there is nothing to post with yet.", "slack_test_refused": "{{detail}}", "slack_test_unavailable": "{{detail}}", "slack_webhook_malformed": "That is not a Slack webhook URL. Copy it from the app's Incoming Webhooks page: it starts with https://hooks.slack.com/services/.", "slack_webhook_required": "Paste the webhook URL: none is saved yet, so there is nothing to keep.", "takedown_already_deleted": "Its entrant already deleted this entry. It is off the board and its archive is deleted.", "takedown_already_taken_down": "This entry is already taken down. Its scorecard says why.", "takedown_reason_missing": "Say why this entry is being removed. The entrant is shown the reason.", "updates_address_malformed": "That is not an email address.", "updates_consent_missing": "Tick the box to say you want these emails.", "updates_full": "Too many people have asked for updates just now, so the list is not taking new addresses for a while. Please try again later.", "updates_not_added": "We could not add you just now. Please try again in a moment.", "updates_not_offered": "Getting updates is not available right now. Please try again later.", "upload_file_unreadable": "That upload could not be read. Send it again." } }