notes/code/misc/python/scripts/textextraction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extracting Text from EPUB Files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: ebooklib in /home/ys/Data/projects/code/Python/Envs/General/lib/python3.11/site-packages (0.18)\n",
      "Requirement already satisfied: beautifulsoup4 in /home/ys/Data/projects/code/Python/Envs/General/lib/python3.11/site-packages (4.12.3)\n",
      "Requirement already satisfied: lxml in /home/ys/Data/projects/code/Python/Envs/General/lib/python3.11/site-packages (5.3.0)\n",
      "Requirement already satisfied: pyperclip in /home/ys/Data/projects/code/Python/Envs/General/lib/python3.11/site-packages (1.9.0)\n",
      "Requirement already satisfied: PyMuPDF in /home/ys/Data/projects/code/Python/Envs/General/lib/python3.11/site-packages (1.24.11)\n",
      "Requirement already satisfied: nltk in /home/ys/Data/projects/code/Python/Envs/General/lib/python3.11/site-packages (3.9.1)\n",
      "Requirement already satisfied: six in /home/ys/Data/projects/code/Python/Envs/General/lib/python3.11/site-packages (from ebooklib) (1.16.0)\n",
      "Requirement already satisfied: soupsieve>1.2 in /home/ys/Data/projects/code/Python/Envs/General/lib/python3.11/site-packages (from beautifulsoup4) (2.5)\n",
      "Requirement already satisfied: click in /home/ys/Data/projects/code/Python/Envs/General/lib/python3.11/site-packages (from nltk) (8.1.7)\n",
      "Requirement already satisfied: joblib in /home/ys/Data/projects/code/Python/Envs/General/lib/python3.11/site-packages (from nltk) (1.4.2)\n",
      "Requirement already satisfied: regex>=2021.8.3 in /home/ys/Data/projects/code/Python/Envs/General/lib/python3.11/site-packages (from nltk) (2024.9.11)\n",
      "Requirement already satisfied: tqdm in /home/ys/Data/projects/code/Python/Envs/General/lib/python3.11/site-packages (from nltk) (4.66.4)\n"
     ]
    }
   ],
   "source": [
    "!python -m pip install ebooklib beautifulsoup4 lxml pyperclip PyMuPDF nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ys/Data/projects/code/Python/Envs/General/lib/python3.11/site-packages/ebooklib/epub.py:1395: UserWarning: In the future version we will turn default option ignore_ncx to True.\n",
      "  warnings.warn('In the future version we will turn default option ignore_ncx to True.')\n",
      "/home/ys/Data/projects/code/Python/Envs/General/lib/python3.11/site-packages/ebooklib/epub.py:1423: FutureWarning: This search incorrectly ignores the root element, and will be fixed in a future version.  If you rely on the current behaviour, change it to './/xmlns:rootfile[@media-type]'\n",
      "  for root_file in tree.findall('//xmlns:rootfile[@media-type]', namespaces={'xmlns': NAMESPACES['CONTAINERNS']}):\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     ForewordBY JAMES K. GALBRAITHTOWARD THE END of this account, my father\n",
      "brought himself, with a show of reluctance that he knew would not be taken too\n",
      "seriously, to comment on whether another crash would come. There were many\n",
      "reasons, including memory and new regulation, why not. But then: No one can\n",
      "doubt that the American people remain susceptible to the speculative mood—to the\n",
      "conviction that enterprise can be attended by unlimited rewards in which they,\n",
      "individually, were meant to share. A rising market can still bring the reality\n",
      "of riches. This, in turn, can draw more and more people to participate. The\n",
      "government preventatives and controls are ready. In the hands of a determined\n",
      "government their efficacy cannot be doubted. There are, however, a hundred\n",
      "reasons why a government will determine not to use them. The main relevance of\n",
      "The Great Crash, 1929 to the Great Crisis of 2008 is surely here. In both cases,\n",
      "the government knew what it should do. Both times, it declined to do it. In the\n",
      "summer of 1929 a few stern words from on high, a rise in the discount rate, a\n",
      "tough investigation into the pyramid schemes of the day, and the house of cards\n",
      "on Wall Street would have tumbled before its fall destroyed the whole economy.\n",
      "In 2004 the FBI warned publicly of \"an epidemic of mortgage fraud.\" But the\n",
      "government did nothing, and less than nothing, delivering instead low interest\n",
      "rates, deregulation, and clear signals that laws would not he enforced. This was\n",
      "fuel for fires. The Greenspan Doctrine held that bubbles cannot be prevented,\n",
      "that the governments task is merely to clean up afterward. The Greenspan\n",
      "practice was to create one bubble after another, until finally one came along so\n",
      "vast that it destroyed the system on the way by. At its source, our crisis is a\n",
      "housing crisis, and not one of too little housing but of too much. It springs\n",
      "from a predatory attack on the safeguards that had for decades kept housing\n",
      "finance safe and stable. In the early 2000s the Bush administration sent clear\n",
      "signals that regulations on mortgages would not be enforced. The signals were\n",
      "not subtle: on one occasion the director of the Office of Thrift Supervision\n",
      "came to a press conference with copies of the Federal Register and a chain saw.\n",
      "There followed every manner of scheme to fleece the unsuspecting—\"liars' loans,\"\n",
      "\"no-doc loans,\" and \"neutron loans\" were terms of art in the business—bundled\n",
      "together, rated and securitized, then spread through the world and left to\n",
      "fester until rising interest rates and crashing prices wrecked the system.\n",
      "Richard Cohen of the Washington Post penned a memorable account of one case, a\n",
      "Marvene Halterman of Avondale, Arizona: At age 61, after 13 years of\n",
      "uninterrupted unemployment and at least as many years of living on welfare, she\n",
      "got a mortgage ... She got it even though at one time she had 23 people living\n",
      "in the house (576 square feet, one bath) and some ramshackle outbuildings. She\n",
      "got it for $103,000, an amount that far exceeded the value of the house. The\n",
      "place has since been condemned ... Halterman's house was never exactly a\n",
      "showcase—the city had since cited her for all the junk (clothes, tires, etc.) on\n",
      "her lawn. Nonetheless, a local financial institution with the cover-your-wallet\n",
      "name of Integrity Funding LLC gave her a mortgage, valuing the house at about\n",
      "twice what a nearby and comparable property sold for ... Integrity Funding then\n",
      "sold the loan to Wells Fargo & Co., which sold it to HSBC Holdings PLC, which\n",
      "then packaged it with thousands of other risky mortgages and offered the\n",
      "indigestible porridge to investors. Standard & Poor's and Moody's Investors\n",
      "Service took a look at it all, as they are supposed to do, and pronounced it\n",
      "\"triple-A.\"  This was fraud, perpetrated in the first instance by the government\n",
      "on the population, and by the rich on the poor. The borrowers were, of course,\n",
      "complicit in some cases, taking out mortgages they never had a hope of paying\n",
      "down. In many more, they were simply naive, gullible, open to pressure,\n",
      "credulous, and hopeful—something might turn up. They took the assurances of\n",
      "lenders that housing prices always rise, that bad loans can always be\n",
      "refinanced. They were attractive to lenders for all of these reasons, and\n",
      "because they had nothing to lose. A borrower with nothing to lose will sign\n",
      "papers that another will not. A government that permits this to happen is\n",
      "complicit in a vast crime.As in 1929, the architects of disaster will form a\n",
      "rich rogues gallery to go shooting in. The Old Objectivist, Alan Greenspan, was\n",
      "intermittently aware of impending disaster and resolutely unwilling to stop it.\n",
      "The Liberal's Banker, Robert Rubin, had a reputation for fiscal probity eclipsed\n",
      "by catastrophic complacency at Citigroup, where he was paid SI 15 million and\n",
      "maintained silence, so far as we know. There will be Phil Gramm, of whom in\n",
      "April 2008 the Washington Post wrote that he was \"the sorcerers apprentice of\n",
      "financial instability and disaster.\" (The Post was quoting me. Reached on the\n",
      "phone, Gramm denied it.) There will be Lawrence Summers, impassioned advocate of\n",
      "the repeal of the Glass-Steagall Act in 1999, bounced from Harvard's presidency\n",
      "to Obama's White House—a man whose reputation remains to be rebuilt or buried by\n",
      "events. And Bernard Madoff. The wreckage of bloated reputations is part of the\n",
      "fun of a crash, at least in deep retrospect. And in the renewed prosperity of\n",
      "the 1950s and later years, my father and history were generally content to leave\n",
      "matters there. Of the figures here told of, Charles Mitchell and Samuel Insull\n",
      "were acquitted, Ivar Kreuger committed suicide, and only Richard Whitney entered\n",
      "Sing-Sing. Juries in our time, if they get the chance, will be less forgiving.\n",
      "Whether they will get the chance is another matter. If they don't, there is not\n",
      "much hope to clean up after the colossal crimes of the past decade. And so far,\n",
      "if the Obama presidency harbors the Roosevelt who gave us the Emergency Banking\n",
      "Act and the SEC, he has not yet come into the open.But there is time, and we\n",
      "shall see.Largely missing from a book on the Great Crisis, 2008, will be the\n",
      "elements of hope, credulity, and carefree optimism that were redeeming features\n",
      "of the 1920s boom. It made people very happy at the time. We read in these pages\n",
      "Frederick Lewis Allen's account of the rich man's chauffeur, his \"ears laid\n",
      "back\" for news of a movement in Bethlehem Steel, and the Wyoming cattleman who\n",
      "traded a thousand shares a day. In 1929, millions thought they could easily\n",
      "become rich, and some did. The parallel moment in modern history came in the\n",
      "late 1990s, in the information-technology boom under President Clinton, which\n",
      "dissolved at the turn of the decade. The years after Bush v. Gore —the years of\n",
      "9/11, the war on terror, and the invasion of Iraq—had no element of this. The\n",
      "masseuse who put her life savings into Madoff's hands wasn't trying to get rich;\n",
      "she was hoping only for a safe and steady return. Millions of others, who took\n",
      "out mortgages, were asking for even less. They were not, in the main,\n",
      "speculators. What they wanted, most of all, was just what everyone else already\n",
      "had: a home of their own. The trouble for them was, there was no way to get one\n",
      "without making a speculative bet. Now they are losing their homes by the\n",
      "millions—an American tragedy, as families double up, move to rentals, crowd into\n",
      "motel rooms or their cars, or spill out into city parks. The victims of the bust\n",
      "extend across the American middle class, to millions of prime credits with half-\n",
      "paid loans and 401(k) plans and a little cash, whose home equity, stock\n",
      "holdings, and interest earnings all collapsed. What in 1930 took the dramatic\n",
      "form of runs on the bank—the wiping out of middle-class savings and wealth—in\n",
      "2008 and 2009 took the form of a simultaneous, month-over-month collapse in\n",
      "asset values, followed by the bankruptcy, downsizing, and liquidation of\n",
      "business firms. In this there was often less immediate hardship—the country is\n",
      "not short of food—than the drying up of possibilities—for college, work, and\n",
      "retirement. This too casts a pall over our time. Just as the years before the\n",
      "recent crisis were not joyful, the world after it risks being less acutely\n",
      "desperate, and more merely dreary, than the age of Roosevelt and the New\n",
      "Deal.Whether the events following the crisis will track the disasters of 1930\n",
      "and later years is also doubtful. In our day of big government—with what\n",
      "economists call automatic stabilizers and immediate fiscal stimulus—slumps are\n",
      "milder and recoveries come sooner than was the case almost a century back. It\n",
      "took four years for the United States to turn to Roosevelt, while by an accident\n",
      "of political timing it was able to elect and install Barack Obama within just a\n",
      "few months. Correspondingly, the situation facing the country today is less\n",
      "dire, the consensus behind radical action is weaker, and the instincts of the\n",
      "administration are less heroic.As this is written, the broad economy seems to\n",
      "have stabilized, and experts are debating whether the stock revival of early\n",
      "2009 is the start of recovery or a \"suckers rally.\" Production may resume, but\n",
      "with an ever larger share coming from imports. Yet things could still go very\n",
      "wrong. The administration seems determined to keep every toxic bank and\n",
      "insurance company alive and intact. No one expects an early revival of\n",
      "employment, and no plans are afoot that would quickly reemploy the millions who\n",
      "are losing their jobs. The Great Crisis has so far not produced its Harry\n",
      "Hopkins, Harold Ickes, Frances Perkins, Henry Wallace, and the others whose\n",
      "eventual emergence is implicit here. For all these reasons, it seems altogether\n",
      "unlikely that the books to come on the Great Crisis of 2008 will be as much fun\n",
      "as this one. A final word. Readers of The Great Crash, 1929 may have wondered\n",
      "about the investing habits of its author, and in particular whether (like\n",
      "Keynes) he was ever tempted by the sirens of financial speculation. I can report\n",
      "that so far as I know he was not. His portfolio was that of a value investor, a\n",
      "buy-and-holder for a retirement he never actually took, as he wrote on, happily\n",
      "and profitably, practically until his death, at ninety-seven, in April 2006.He\n",
      "was also not averse, on occasion, to even stricter precaution. I remember\n",
      "calling him on the night of the market break in October 1987, an event that\n",
      "brought this book back from a brief disappearance. It was, due to the attentions\n",
      "of the national press, somewhat difficult to get through on the phone. But when\n",
      "I did, I heard the reassuring paternal voice:\"James? Is that you?\" Pause. \"Not\n",
      "to worry: I've been in cash for three weeks.\"But then there was another pause,\n",
      "and the tone changed.\"But I'm sorry to say, the same cannot be said of your\n",
      "mother. She finds it very difficult to sell the General Electric that her family\n",
      "bought from Edison for a dollar.\"AUSTIN, TEXAS MAY 18, 2009\n",
      "IntroductionThe View from the NinetiesThe Great Crash, 1929 was first published\n",
      "in 1955 and has been continuously in print ever since, a matter now of forty\n",
      "years and more. Authors (and publishers) being as they are, the tendency is to\n",
      "attribute this endurance to the excellence of the work. Evidently this book has\n",
      "some merit, but, for worse or perhaps better, there is another reason for its\n",
      "durability. Each time it has been about to pass from print and the bookstores,\n",
      "another speculative episode—another bubble or the ensuing misfortune—has stirred\n",
      "interest in the history of this, the great modern case of boom and collapse,\n",
      "which led on to an unforgiving depression.One of the subsequent episodes\n",
      "occurred, in fact, as the book was coming from the printer. There was a small\n",
      "stock market boom in the spring of 1955; I was called to Washington to testily\n",
      "at a Senate hearing on the past experience. During my testimony that morning the\n",
      "stock market went suddenly south. I was blamed for the collapse, especially by\n",
      "those who were long in the market. A fair number of the latter wrote to threaten\n",
      "me with physical injury; a more devout citizenry told me they were praying for\n",
      "my ill health or early demise. A few days after my testimony I broke my leg\n",
      "while skiing in Vermont. The papers carried mention. Letters came in telling me\n",
      "of prayers that had been answered. I had at least done something for religion.\n",
      "In the mood of the times a senator from Indiana, Homer E. Capehart, said it was\n",
      "the work of a crypto-Communist.That was only the beginning. The offshore-funds\n",
      "insanity of the seventies, the big bust of 1987, less dramatic episodes or\n",
      "fears, all brought attention back to 1929 and kept the book in print. And so\n",
      "again now in 1997. That we are having a major speculative splurge as this is\n",
      "written is obvious to anyone not captured by vacuous optimism. There is now far\n",
      "more money flowing into the stock markets than there is intelligence to guide\n",
      "it. There are many more mutual funds than there are financially acute,\n",
      "historically aware men and women to manage them. I am not given to prediction;\n",
      "one's foresight is forgotten, only one's errors are well remembered. But there\n",
      "is here a basic and recurrent process. It comes with rising prices, whether of\n",
      "stocks, real estate, works of art or anything else. This increase attracts\n",
      "attention and buyers, which produces the further effect of even higher prices.\n",
      "Expectations are thus justified by the very action that sends prices up. The\n",
      "process continues; optimism with its market effect is the order of the day.\n",
      "Prices go up even more. Then, for reasons that will endlessly be debated, comes\n",
      "the end. The descent is always more sudden than the increase; a balloon that has\n",
      "been punctured does not deflate in an orderly way.To repeat, I make no\n",
      "prediction; I only observe that this phenomenon has manifested itself many times\n",
      "since 1637, when Dutch speculators saw tulip bulbs as their magic road to\n",
      "wealth, and 1720, when John Law brought presumptive wealth and then sudden\n",
      "poverty to Paris through the pursuit of gold, to this day undiscovered, in\n",
      "Louisiana. In these years also the great South Sea Bubble spread financial\n",
      "devastation in Britain.Later there was more. In the United States in the\n",
      "nineteenth century there was a speculative splurge every twenty or thirty years.\n",
      "This was already a tradition, for the colonies, north and south, had\n",
      "experimented at no slight eventual cost with currency issues that had no visible\n",
      "backing. They did well until it was observed that there was nothing there. The\n",
      "Revolution was paid for with Continental notes, giving permanence to the phrase\n",
      "\"not worth a Continental.\" In the years following the war of 1812–14, there was\n",
      "a major real estate boom; in the 1830s came wild speculation in canal and\n",
      "turnpike investment—internal improvements, they were termed. Along with this\n",
      "went issues of bank notes unbacked by anything of value and issued by anyone\n",
      "able to hire a building larger than that of the local blacksmith. This came\n",
      "powerfully to an end in 1837. In the 1850s came another boom and collapse, and\n",
      "in those years a New England bank, in a part of the country more cautious than\n",
      "most, closed down. It had $500,000 in notes outstanding and assets to cover them\n",
      "of $86.48. After the Civil War came the railroad boom and a particularly painful\n",
      "collapse in 1873. Another boom came to an equally dramatic end in 1907, but the\n",
      "big New York banks were able, this time, to limit the damage. Earlier a\n",
      "considerable flow of British funds had fueled the American speculation, notably\n",
      "that just mentioned in railroads. There was also a renewed British involvement\n",
      "in South America, the South Sea Bubble now forgotten. The greatly distinguished\n",
      "Baring Brothers had to be rescued by the Bank of England from bankruptcy\n",
      "occasioned by its loans to Argentina. This is currently interesting, for in the\n",
      "1990s Barings was caught up in the more or less incredible operations of one of\n",
      "its minor minions in Singapore. This time there was no rescue; Barings, for all\n",
      "public purposes, disappeared.If we do now have a downturn—what is called a day\n",
      "of reckoning—some things can, indeed, be foreseen. By some estimates a quarter\n",
      "of all Americans, directly or indirectly, are in the stock market. Were there a\n",
      "bad slump, it would limit their expenditures, especially of durable goods, and\n",
      "put pressure on their very large credit card debt. The result would be a\n",
      "generally adverse effect on the economy. This would not be as painful as the\n",
      "aftereffects of 1929; then banks were fragile and without deposit insurance,\n",
      "farm markets were important and especially vulnerable, there was no cushioning\n",
      "effect from unemployment compensation, welfare payments and Social Security. All\n",
      "this is better now. But there could be a recession; that would be normal. There\n",
      "would also be, we may be certain, the traditional reassuring words from\n",
      "Washington. Always when markets are in trouble, the phrases are the same: \"The\n",
      "economic situation is fundamentally sound\" or simply \"The fundamentals are\n",
      "good.\" All who hear these words should know that something is wrong. Once more I\n",
      "do not predict and tell only what the past so vividly tells us. I offer a final\n",
      "word on this book. It was published in that spring of 1955 to an appreciative\n",
      "audience. There was a brief appearance on the best-seller lists; I looked with\n",
      "pleasure at the bookstore windows. On my frequent visits to New York, I was\n",
      "distressed, however, to see no sign of it in a small bookshop on the ramp\n",
      "leading down to the planes in the old La Guardia terminal. One night I stopped\n",
      "in to inspect the shelves. The lady in charge finally noticed me and asked what\n",
      "I sought. Somewhat embarrassed, I passed over the name of the author and said it\n",
      "was a work called The Great Crash. \"Not a book you could sell in an airport,\"\n",
      "she responded firmly.         A Note on SourcesIn recent times numerous authors\n",
      "and publishers have come to suppose that readers are offended by footnotes. I\n",
      "have no desire to offend or even in the slightest way to discourage any solvent\n",
      "customer, but I regard this supposition as silly. No literate person can\n",
      "possibly be disturbed by a little small type at the bottom of a page, and\n",
      "everyone, professional and lay reader alike, needs to know on occasion the\n",
      "credentials of a fact. Footnotes also provide an exceedingly good index of the\n",
      "care with which a subject has been researched.However, there is also a line\n",
      "between adequacy and pedantry. In this book, where I have drawn on public\n",
      "documents, books, magazine articles, or special sources of any kind I have\n",
      "indicated the source. However, much of the story of 1929 is to be found in the\n",
      "general and financial press of the time. Systematic citation of these sources\n",
      "would involve endless references to the same papers. This I have not done. It\n",
      "means in general that if no source is given, the reader can assume it was in the\n",
      "New York Times, the Wall Street Journal, and the other papers of general\n",
      "circulation of the day.         CHAPTER I\"Vision and Boundless Hope and\n",
      "Optimism\"ON DECEMBER 4, 1928, President Coolidge sent his last message on the\n",
      "state of the Union to the reconvening Congress. Even the most melancholy\n",
      "congressman must have found reassurance in his words. \"No Congress of the United\n",
      "States ever assembled, on surveying the state of the Union, has met with a more\n",
      "pleasing prospect than that which appears at the present time. In the domestic\n",
      "field there is tranquility and contentment ... and the highest record of years\n",
      "of prosperity. In the foreign field there is peace, the goodwill which comes\n",
      "from mutual understanding ...\" He told the legislators that they and the country\n",
      "might \"regard the present with satisfaction and anticipate the future with\n",
      "optimism.\" And breaking sharply with the most ancient of our political\n",
      "conventions, he omitted to attribute this well-being to the excellence of the\n",
      "administration which he headed. \"The main source of these unexampled blessings\n",
      "lies in the integrity and character of the American people.\"A whole generation\n",
      "of historians has assailed Coolidge for the superficial optimism which kept him\n",
      "from seeing that a great storm was brewing at home and also more distantly\n",
      "abroad. This is grossly unfair. It requires neither courage nor prescience to\n",
      "predict disaster. Courage is required of the man who, when things are good, says\n",
      "so. Historians rejoice in crucifying the false prophet of the millennium. They\n",
      "never dwell on the mistake of the man who wrongly predicted Armageddon. There\n",
      "was much that was good about the world of which Coolidge spoke. True, as liberal\n",
      "misanthropes have insisted, the rich were getting richer much faster than the\n",
      "poor were getting less poor. The farmers were unhappy and had been ever since\n",
      "the depression of 1920–21 had cut farm prices sharply but left costs high. Black\n",
      "people in the South and white people in the southern Appalachians continued to\n",
      "dwell in hopeless poverty. Fine old-English houses with high gables, leaded\n",
      "glass, and well-simulated half-timbering were rising in the country club\n",
      "district, while farther in town one encountered the most noisome slums outside\n",
      "the Orient.All this notwithstanding, the twenties in America were a very good\n",
      "time. Production and employment were high and rising. Wages were not going up\n",
      "much, but prices were stable. Although many people were still very poor, more\n",
      "people were comfortably well-off, well-to-do, or rich than ever before. Finally,\n",
      "American capitalism was undoubtedly in a lively phase. Between 1925 and 1929,\n",
      "the number of manufacturing establishments increased from 183,900 to 206,700;\n",
      "the value of their output rose from $60.8 billions to $68.0 billions.1 The\n",
      "Federal Reserve index of industrial production which had averaged only 67 in\n",
      "1921 (1923–25= 100) had risen to 110 by July 1928, and it reached 126 in June\n",
      "1929.2 In 1926, 4,301,000 automobiles were produced. Three years later, in 1929,\n",
      "production had increased by over a million to 5,358,000,3 a figure which\n",
      "compares very decently with the 5,700,000 new car registrations of the opulent\n",
      "year of 1953. Business earnings were rising rapidly, and it was a good time to\n",
      "be in business. Indeed, even the most jaundiced histories of the era concede,\n",
      "tacitly, that times were good, for they nearly all join in taxing Coolidge for\n",
      "his failure to see that they were too good to last.This notion of an iron law of\n",
      "compensation—the notion that the ten good years of the twenties had to be paid\n",
      "for by the ten bad ones of the thirties—is one to which it will be worthwhile to\n",
      "return.         IIOne thing in the twenties should have been visible even to\n",
      "Coolidge. It concerned the American people of whose character he had spoken so\n",
      "well. Along with the sterling qualities he praised, they were also displaying an\n",
      "inordinate desire to get rich quickly with a minimum of physical effort. The\n",
      "first striking manifestation of this personality trait was in Florida. There, in\n",
      "the mid-twenties, Miami, Miami Beach, Coral Gables, the East Coast as far north\n",
      "as Palm Beach, and the cities over on the Gulf had been struck by the great\n",
      "Florida real estate boom. The Florida boom contained all of the elements of the\n",
      "classic speculative bubble. There was the indispensable element of substance.\n",
      "Florida had a better winter climate than New York, Chicago, or Minneapolis.\n",
      "Higher incomes and better transportation were making it increasingly accessible\n",
      "to the frost-bound North. The time indeed was coming when the annual flight to\n",
      "the South would be as regular and impressive as the migrations of the Canada\n",
      "Goose.On that indispensable element of fact men and women had proceeded to build\n",
      "a world of speculative make-believe. This is a world inhabited not by people who\n",
      "have to be persuaded to believe but by people who want an excuse to believe. In\n",
      "the case of Florida, they wanted to believe that the whole peninsula would soon\n",
      "be populated by the holiday-makers and the sun-worshippers of a new and\n",
      "remarkably indolent era. So great would be the crush that beaches, bogs, swamps,\n",
      "and common scrubland would all have value. The Florida climate obviously did not\n",
      "insure that this would happen. But it did enable people who wanted to believe it\n",
      "would happen so to believe. However, speculation does not depend entirely on the\n",
      "capacity for self-delusion. In Florida land was divided into building lots and\n",
      "sold for a 10 per cent down payment. Palpably, much of the unlovely terrain that\n",
      "thus changed hands was as repugnant to the people who bought it as to the\n",
      "passer-by. The buyers did not expect to live on it; it was not easy to suppose\n",
      "that anyone ever would. But these were academic considerations. The reality was\n",
      "that this dubious asset was gaining in value by the day and could be sold at a\n",
      "handsome profit in a fortnight. It is another feature of the speculative mood\n",
      "that, as time passes, the tendency to look beyond the simple fact of increasing\n",
      "values to the reasons on which it depends greatly diminishes. And there is no\n",
      "reason why anyone should do so as long as the supply of people who buy with the\n",
      "expectation of selling at a profit continues to be augmented at a sufficiently\n",
      "rapid rate to keep prices rising.Through 1925 the pursuit of effortless riches\n",
      "brought people to Florida in satisfactorily increasing numbers. More land was\n",
      "subdivided each week. What was loosely called seashore became five, ten, or\n",
      "fifteen miles from the nearest brine. Suburbs became an astonishing distance\n",
      "from town. As the speculation spread northward, an enterprising Bostonian, Mr.\n",
      "Charles Ponzi, developed a subdivision \"near Jacksonville.\" It was approximately\n",
      "sixty-five miles west of the city. (In other respects Ponzi believed in good,\n",
      "compact neighborhoods; he sold twenty-three lots to the acre.) In instances\n",
      "where the subdivision was close to town, as in the case of Manhattan Estates,\n",
      "which were \"not more than three fourths of a mile from the prosperous and fast-\n",
      "growing city of Nettie,\" the city, as was so of Nettie, did not exist. The\n",
      "congestion of traffic into the state became so severe that in the autumn of 1925\n",
      "the railroads were forced to proclaim an embargo on less essential freight,\n",
      "which included building materials for developing the subdivisions. Values rose\n",
      "wonderfully. Within forty miles of Miami \"inside\" lots sold at from $8000 to\n",
      "$20,000; waterfront lots brought from $15,000 to $25,000, and more or less bona\n",
      "fide seashore sites brought $20,000 to $75,000.4 However, in the spring of 1926,\n",
      "the supply of new buyers, so essential to the reality of increasing prices,\n",
      "began to fail. As 1928 and 1929 were to show, the momentum built up by a good\n",
      "boom is not dissipated in a moment. For a while in 1926 the increasing eloquence\n",
      "of the promoters offset the diminishing supply of prospects. (Even the cathedral\n",
      "voice of William Jennings Bryan, which once had thundered against the cross of\n",
      "gold, had been for a time enlisted in the sorry task of selling swampland.) But\n",
      "this boom was not left to collapse of its own weight. In the autumn of 1926, two\n",
      "hurricanes showed, in the words of Frederick Lewis Allen, \"what a Soothing\n",
      "Tropic Wind could do when it got a running start from the West Indies.\" 5 The\n",
      "worst of these winds, on September 18, 1926, killed four hundred people, tore\n",
      "the roofs from thousands of houses, and piled tons of water and a number of\n",
      "elegant yachts into the streets of Miami. There was agreement that the storm had\n",
      "caused a healthy breathing spell in the boom, although its resumption was\n",
      "predicted daily. In the Wall Street Journal of October 8, 1926, one Peter O.\n",
      "Knight, an official of the Seaboard Air Line and a sincere believer in the\n",
      "future of Florida, acknowledged that some seventeen or eighteen thousand people\n",
      "were in need of assistance. But he added: \"The same Florida is still there with\n",
      "its magnificent resources, its wonderful climate, and its geographical position.\n",
      "It is the Riviera of America.\" He expressed concern that the solicitation of Red\n",
      "Cross funds for hurricane relief would \"do more damage permanently to Florida\n",
      "than would be offset by the funds received.\"6 This reluctance to concede that\n",
      "the end has come is also in accordance with the classic pattern. The end had\n",
      "come in Florida. In 1925 bank clearings in Miami were $1,066,- 528,000; by 1928\n",
      "they were down to $143,364,000.7 Farmers who had sold their land at a handsome\n",
      "price and had condemned themselves as it later sold for double, treble,\n",
      "quadruple the original price, now on occasion got it back through a whole chain\n",
      "of subsequent defaults. Sometimes it was equipped with eloquently named streets\n",
      "and with sidewalks, street lamps, and taxes and assessments amounting to several\n",
      "times its current value.The Florida boom was the first indication of the mood of\n",
      "the twenties and the conviction that God intended the American middle class to\n",
      "be rich. But that this mood survived the Florida collapse is still more\n",
      "remarkable. It was widely understood that things had gone to pieces in Florida.\n",
      "While the number of speculators was almost certainly small compared with the\n",
      "subsequent participation in the stock market, nearly every community contained a\n",
      "man who was known to have taken \"quite a beating\" in Florida. For a century\n",
      "after the collapse of the South Sea Bubble, Englishmen regarded the most\n",
      "reputable joint stock companies with some suspicion. Even as the Florida boom\n",
      "collapsed, the faith of Americans in quick, effortless enrichment in the stock\n",
      "market was becoming every day more evident.         IIIIt is hard to say when\n",
      "the stock market boom of the nineteen-twenties began. There were sound reasons\n",
      "why, during these years, the prices of common stocks should rise. Corporate\n",
      "earnings were good and growing. The prospect seemed benign. In the early\n",
      "twenties stock prices were low and yields favorable.In the last six months of\n",
      "1924, the prices of securities began to rise, and the increase was continued and\n",
      "extended through 1925. Thus at the end of May 1924, the New York Times average\n",
      "of the prices of twenty-five industrial stocks was 106; by the end of the year\n",
      "it was 134.8 By December 31, 1925, it had gained very nearly another 50 points\n",
      "and stood at 181. The advance through 1925 was remarkably steady; there were\n",
      "only a couple of months when values did not show a net gain. During 1926 there\n",
      "was something of a setback. Business was off a little in the early part of that\n",
      "year; it was thought by many that values the year before had risen unreasonably.\n",
      "February brought a sharp fall in the market, and March a rather abrupt collapse.\n",
      "The Times industrials went down from 181 at the beginning of the year to 172 at\n",
      "the end of February, and then dropped by nearly 30 points to 143 at the end of\n",
      "March. However, in April the market steadied and renewed its advance. Another\n",
      "mild setback occurred in October, just after the hurricane blew away the\n",
      "vestiges of the Florida boom, but again recovery was prompt. At the end of the\n",
      "year values were about where they had been at the beginning.In 1927 the increase\n",
      "began in earnest. Day after day and month after month the price of stocks went\n",
      "up. The gains by later standards were not large, but they had an aspect of great\n",
      "reliability. Again in only two months in 1927 did the averages fail to show an\n",
      "increase. On May 20, when Lindbergh took off from Roosevelt Field and headed for\n",
      "Paris, a fair number of citizens were unaware of the event. The market, which\n",
      "that day was registering another of its small but solid gains, had by then\n",
      "acquired a faithful band of devotees who spared no attention for more celestial\n",
      "matters.In the summer of 1927 Henry Ford rang down the curtain on the immortal\n",
      "Model T and closed his plant to prepare for Model A. The Federal Reserve index\n",
      "of industrial production receded, presumably as a result of the Ford shutdown,\n",
      "and there was general talk of depression. The effect on the market was\n",
      "imperceptible. At the end of the year, by which time production had also turned\n",
      "up again, the Times industrials had reached 245, a net gain of 69 points for the\n",
      "year. The year 1927 is historic from another point of view in the lore of the\n",
      "stock market. According to a long accepted doctrine, it was in this year that\n",
      "the seeds of the eventual disaster were sown. The responsibility rests with an\n",
      "act of generous but ill-advised internationalism. Some—including Mr. Hoover—have\n",
      "thought it almost disloyal, although in those days accusations of treason were\n",
      "still made with some caution.In 1925, under the aegis of the then Chancellor of\n",
      "the Exchequer, Mr. Winston Churchill, Britain returned to the gold standard at\n",
      "the old or pre-World War I relationship between gold, dollars, and the pound.\n",
      "There is no doubt that Churchill was more impressed by the grandeur of the\n",
      "traditional, or $4.86, pound than by the more subtle consequences of\n",
      "overvaluation, which he is widely assumed not to have understood. The\n",
      "consequences, nonetheless, were real and severe. Customers of Britain had now to\n",
      "use these costly pounds to buy goods at prices that still reflected wartime\n",
      "inflation. Britain was, accordingly, an unattractive place for foreigners to\n",
      "buy. For the same reason it was an easy place in which to sell. In 1925 began\n",
      "the long series of exchange crises which, like the lions in Trafalgar Square and\n",
      "the street walkers in Piccadilly, are now an established part of the British\n",
      "scene. There were also unpleasant domestic consequences; the bad market for coal\n",
      "and the effort to reduce costs and prices to meet world competition led to the\n",
      "general strike in 1926.Then, as since, gold when it escaped from Britain or\n",
      "Europe came to the United States. This might be discouraged if prices of goods\n",
      "were high and interest rates were low in this country. (The United States would\n",
      "be a poor place in which to buy and invest.) In the spring of 1927, three august\n",
      "pilgrims—Montagu Norman, the Governor of the Bank of England, the durable\n",
      "Hjalmar Schacht, then Governor of the Reichsbank, and Charles Rist, the Deputy\n",
      "Governor of the Bank of France—came to the United States to urge an easy money\n",
      "policy. (They had previously pled with success for a roughly similar policy in\n",
      "1925.) The Federal Reserve obliged. The rediscount rate of the New York Federal\n",
      "Reserve Bank was cut from 4 to 3.5 per cent. Government securities were\n",
      "purchased in considerable volume with the mathematical consequence of leaving\n",
      "the banks and individuals who had sold them with money to spare. Adolph C.\n",
      "Miller, a dissenting member of the Federal Reserve Board, subsequently described\n",
      "this as \"the greatest and boldest operation ever undertaken by the Federal\n",
      "Reserve System, and...[it] resulted in one of the most costly errors committed\n",
      "by it or any other banking system in the last 75 years!\"9 The funds that the\n",
      "Federal Reserve made available were either invested in common stocks or (and\n",
      "more important) they became available to help finance the purchase of common\n",
      "stocks by others. So provided with funds, people rushed into the market. Perhaps\n",
      "the most widely read of all the interpretations of the period, that of Professor\n",
      "Lionel Robbins of the London School of Economics, concludes: \"From that date,\n",
      "according to all the evidence, the situation got completely out of control.\"10\n",
      "This view that the action of the Federal Reserve authorities in 1927 was\n",
      "responsible for the speculation and collapse which followed has never been\n",
      "seriously shaken. There are reasons why it is attractive. It is simple, and it\n",
      "exonerates both the American people and their economic system from any\n",
      "substantial blame. The danger of being guided by foreigners is well known, and\n",
      "Norman and Schacht had some special reputation for sinister motives.Yet the\n",
      "explanation obviously assumes that people will always speculate if only they can\n",
      "get the money to finance it. Nothing could be farther from the case. There were\n",
      "times before and there have been long periods since when credit was plentiful\n",
      "and cheap—far cheaper than in 1927–29—and when speculation was negligible. Nor,\n",
      "as we shall see later, was speculation out of control after 1927, except that it\n",
      "was beyond the reach of men who did not want in the least to control it. The\n",
      "explanation is a tribute only to a recurrent preference, in economic matters,\n",
      "for formidable nonsense.         IVUntil the beginning of 1928, even a man of\n",
      "conservative mind could believe that the prices of common stock were catching up\n",
      "with the increase in corporation earnings, the prospect for further increases,\n",
      "the peace and tranquility of the times, and the certainty that the\n",
      "Administration then firmly in power in Washington would take no more than\n",
      "necessary of any earnings in taxes. Early in 1928, the nature of the boom\n",
      "changed. The mass escape into make-believe, so much a part of the true\n",
      "speculative orgy, started in earnest. It was still necessary to reassure those\n",
      "who required some tie, however tenuous, to reality. And, as will be seen\n",
      "presently, this process of reassurance—of inventing the industrial equivalents\n",
      "of the Florida climate—eventually achieved the status of a profession. However,\n",
      "the time had come, as in all periods of speculation, when men sought not to be\n",
      "persuaded of the reality of things but to find excuses for escaping into the new\n",
      "world of fantasy. There were many indications by 1928 that this phase had come.\n",
      "Most obvious was the behavior of the market. While the winter months of 1928\n",
      "were rather quiet, thereafter the market began to rise, not by slow, steady\n",
      "steps, but by great vaulting leaps. On occasion it also came down the same way,\n",
      "only to recover and go higher again. In March 1928 the industrial average rose\n",
      "nearly 25 points. News of the boiling market was frequently on the front page.\n",
      "Individual issues sometimes made gains of 10, 15, and 20 points in a single\n",
      "day's trading. On March 12, Radio, in many respects the speculative symbol of\n",
      "the time, gained 18 points. On the following day it opened 22 points above the\n",
      "previous close. Then it lost 20 points on the announcement that the behavior of\n",
      "the trading in the stock was being investigated by the Exchange, gained 15\n",
      "points, and fell off 9.11 A few days later, on a strong market, it made another\n",
      "18-point gain.The March boom also celebrated, beyond anything theretofore, the\n",
      "operations of the big professional traders. The lore of competitive markets\n",
      "pictures the stock exchange as the most impersonal of markets. No doctrine is\n",
      "more jealously guarded by the prophets and defenders of the Stock Exchange. \"The\n",
      "Exchange is a market place where prices reflect the basic law of supply and\n",
      "demand,\" the New York Stock Exchange says firmly of itself.12 Yet even the most\n",
      "devout Wall Streeter allows himself on occasion to believe that more personal\n",
      "influences have a hand in his destiny. Somewhere around there are big men who\n",
      "put stocks up and put them down. As the boom developed, the big men became more\n",
      "and more omnipotent in the popular or at least in the speculative view. In\n",
      "March, according to this view, the big men decided to put the market up, and\n",
      "even some serious scholars have been inclined to think that a concerted move\n",
      "catalyzed this upsurge. If so, the important figure was John J. Raskob. Raskob\n",
      "had impressive associations. He was a director of General Motors, an ally of the\n",
      "Du Ponts and soon to be Chairman of the Democratic National Committee by choice\n",
      "of Al Smith. A contemporary student of the market, Professor Charles Amos Dice\n",
      "of the Ohio State University, thought this latter appointment a particular\n",
      "indication of the new prestige of Wall Street and the esteem in which it was\n",
      "held by the American people. \"Today,\" he observed, \"the shrewd, worldly-wise\n",
      "candidate of one of the great political parties chooses one of the outstanding\n",
      "operators in the stock market ... as a goodwill creator and popular vote\n",
      "getter.\" 13On March 23, 1928, on taking ship for Europe, Raskob spoke favorably\n",
      "of prospects for automobile sales for the rest of the year and of the share in\n",
      "the business that General Motors would have. He may also have suggested—the\n",
      "evidence is not entirely clear—that G.M. stock should be selling at not less\n",
      "than twelve times earnings. This would have meant a price of 225 as compared\n",
      "with a current quotation of about 187. Such, as the Times put it, was \"the magic\n",
      "of his name\" that Mr. Raskob's \"temperate bit of optimism\" sent the market into\n",
      "a boiling fury. On March 24, a Saturday, General Motors gained nearly 5 points,\n",
      "and the Monday following it went to 199. The surge in General Motors, meanwhile,\n",
      "set off a great burst of trading elsewhere in the list. Among the others who\n",
      "were assumed to have put their strength behind the market that spring was\n",
      "William Crapo Durant. Durant was the organizer of General Motors, whom Raskob\n",
      "and the Du Ponts had thrown out of the company in 1920. After a further\n",
      "adventure in the auto business, he had turned to full-time speculation in the\n",
      "stock market. The seven Fisher brothers were also believed to be influential.\n",
      "They too were General Motors alumni and had come to Wall Street with the great\n",
      "fortune they had realized from the sale of the Fisher-body plants. Still another\n",
      "was Arthur W. Cutten, the Canadian-born grain speculator who had recently\n",
      "shifted his market operations to Wall Street from the Chicago Board of Trade. As\n",
      "a market operator, Cutten surmounted substantial personal handicaps. He was very\n",
      "hard of hearing, and some years later, before a congressional committee, even\n",
      "his own counsel conceded that his memory was very defective.Observing this group\n",
      "as a whole Professor Dice was especially struck by their \"vision for the future\n",
      "and boundless hope and optimism.\" He noted that \"they did not come into the\n",
      "market hampered by the heavy armor of tradition.\" In recounting their effect on\n",
      "the market, Professor Dice obviously found the English language verging on\n",
      "inadequacy. \"Led by these mighty knights of the automobile industry, the steel\n",
      "industry, the radio industry...\" he said, \"and finally joined, in despair, by\n",
      "many professional traders who, after much sack-cloth and ashes, had caught the\n",
      "vision of progress, the Coolidge market had gone forward like the phalanxes of\n",
      "Cyrus, parasang upon parasang and again parasang upon parasang...\"14         VIn\n",
      "June of 1928 the market retreated a parasang or two—in fact, the losses during\n",
      "the first three weeks were almost as great as the March gains. June 12, a day of\n",
      "particularly heavy losses, was a landmark. For a year or more, men of vision had\n",
      "been saying that the day might come when five million shares would be traded on\n",
      "the New York Stock Exchange. Once this had been only a wild conversational\n",
      "gambit, but for some time it had shown signs of being overtaken by the reality.\n",
      "On March 12, the volume of trading had reached 3,875,910 shares, an all-time\n",
      "high. By the end of the month such a volume had become commonplace. On March 27,\n",
      "4,790,270 shares were traded. Then on June 12, 5,052, 790 shares changed hands.\n",
      "The ticker also fell nearly two hours behind the market; Radio dropped 23\n",
      "points, and a New York paper began its accounts of the day's events, \"Wall\n",
      "Street's bull market collapsed yesterday with a detonation heard round the\n",
      "world.\"The announcement of the death of the bull market was as premature as any\n",
      "since that of the death of Mark Twain. In July there was a small net gain, and\n",
      "in August a strong upsurge. Thereafter not even the approach of the election\n",
      "caused serious hesitation. People remained unperturbed when, on September 17,\n",
      "Roger W. Babson told an audience in Wellesley, Massachusetts, that \"if Smith\n",
      "should be elected with a Democratic Congress we are almost certain to have a\n",
      "resulting business depression in 1929.\" He also said that \"the election of\n",
      "Hoover and a Republican Congress should result in continued prosperity for\n",
      "1929,\" and it may have been that the public knew it would be Hoover. In any\n",
      "case, during the same month reassurance came from still higher authority. Andrew\n",
      "W. Mellon said, \"There is no cause for worry. The high tide of prosperity will\n",
      "continue.\"Mr. Mellon did not know. Neither did any of the other public figures\n",
      "who then, as since, made similar statements. These are not forecasts; it is not\n",
      "to be supposed that the men who make them are privileged to look farther into\n",
      "the future than the rest. Mr. Mellon was participating in a ritual which, in our\n",
      "society, is thought to be of great value for influencing the course of the\n",
      "business cycle. By affirming solemnly that prosperity will continue, it is\n",
      "believed, one can help insure that prosperity will in fact continue. Especially\n",
      "among businessmen the faith in the efficiency of such incantation is very great.\n",
      "VIHoover was elected in a landslide. This, were the speculators privy to Mr.\n",
      "Hoover's mind, should have caused a heavy fall in the market. In his memoirs Mr.\n",
      "Hoover states that as early as 1925 he became concerned over the \"growing tide\n",
      "of speculation.\" 15 During the months and years that followed this concern\n",
      "gradually changed to alarm, and then to something only slightly less than a\n",
      "premonition of total disaster. \"There are crimes,\" Mr. Hoover said of\n",
      "speculation, \"far worse than murder for which men should be reviled and\n",
      "punished.\"16 As Secretary of Commerce he had sought nothing so much as to get\n",
      "the market under control.Mr. Hoover's attitude toward the market was, however,\n",
      "an exceptionally well-kept secret. People did not know of his efforts, uniformly\n",
      "frustrated by Coolidge and the Federal Reserve Board, to translate his thoughts\n",
      "into action. The news of his election, so far from causing a panic, set off the\n",
      "greatest increase in buying to date. On November 7, the day after the election,\n",
      "there was a \"victory boom,\" and the market leaders climbed 5 to 15 points.\n",
      "Volume reached 4,894,670 shares, or only a little less than the all-time record\n",
      "of June 12, and this new level was reached on a rising, not a falling market. On\n",
      "November 16, a further wave of buying hit the market. An astonishing 6,641,250\n",
      "shares changed hands—far above the previous record. The Times industrial\n",
      "averages made a net gain of 4½ points on the day's trading—then considered an\n",
      "impressive advance. Apart from the afterglow of the election, there was nothing\n",
      "particular to incite this enthusiasm. The headlines of the day told only of the\n",
      "sinking of the steamship Vestris and the epic achievements of the officers and\n",
      "crew in shouldering aside the women and children and saving their own lives.\n",
      "November 20 was another huge day. Trading—6,503,230 shares—was fractionally\n",
      "smaller than on the sixteenth, but by common agreement it was much more frantic.\n",
      "The following morning the Times observed that \"for cyclonic violence yesterday's\n",
      "stock market has never been exceeded in the history of Wall Street.\" December\n",
      "was not so good. Early in the month there was a bad break, and, on December 8,\n",
      "Radio fell a ghastly 72 points in one day. However, the market steadied and then\n",
      "came back. Over the whole year of 1928 the Times industrial average gained 86\n",
      "points, or from 245 to 331. During the year Radio went from 85 to 420 (it had\n",
      "never paid a dividend); Du Pont went from 310 to 525; Montgomery Ward from 117\n",
      "to 440; Wright Aeronautic from 69 to 289.17 During the year 920,550,032 shares\n",
      "were traded on the New York Stock Exchange, as compared with a record-breaking\n",
      "576,990,875 in 1927.18 But there was still another and even more significant\n",
      "index of what was happening in the market. That was the phenomenal increase in\n",
      "trading on margin.         VIIAs noted, at some point in the growth of a boom\n",
      "all aspects of property ownership become irrelevant except the prospect for an\n",
      "early rise in price. Income from the property, or enjoyment of its use, or even\n",
      "its long-run worth is now academic. As in the case of the more repulsive Florida\n",
      "lots, these usufructs may be non-existent or even negative. What is important is\n",
      "that tomorrow or next week market values will rise—as they did yesterday or last\n",
      "week—and a profit can be realized.It follows that the only reward to ownership\n",
      "in which the boomtime owner has an interest is the increase in values. Could the\n",
      "right to the increased value be somehow divorced from the other and now\n",
      "unimportant fruits of possession and also from as many as possible of the\n",
      "burdens of ownership, this would be much welcomed by the speculator. Such an\n",
      "arrangement would enable him to concentrate on speculation which, after all, is\n",
      "the business of a speculator.Such is the genius of capitalism that where a real\n",
      "demand exists it does not go long unfilled. In all great speculative orgies\n",
      "devices have appeared to enable the speculator so to concentrate on his\n",
      "business. In the Florida boom the trading was in \"binders.\" Not the land itself\n",
      "but the right to buy the land at a stated price was traded. This right to\n",
      "buy—which was obtained by a down payment of 10 per cent of the purchase\n",
      "price—could be sold. It thus conferred on the speculators the full benefit of\n",
      "the increase in values. After the value of the lot had risen he could resell the\n",
      "binder for what he had paid plus the full amount of the increase in price. The\n",
      "worst of the burdens of ownership, whether of land or any other asset, is the\n",
      "need to put up the cash represented by the purchase price. The use of the binder\n",
      "cut this burden by 90 per cent—or it multiplied tenfold the amount of acreage\n",
      "from which the speculator could harvest an increase in value. The buyer happily\n",
      "gave up the other advantages of ownership. These included the current income of\n",
      "which, invariably, there was none and the prospect of permanent use in which he\n",
      "had not the slightest interest.The stock market also has its design for\n",
      "concentrating the speculative energies of the speculator, and, as might be\n",
      "expected, it improves substantially on the crudities of the real estate market.\n",
      "In the stock market the buyer of securities on margin gets full title to his\n",
      "property in an unconditional sale. But he rids himself of the most grievous\n",
      "burden of ownership—that of putting up the purchase price—by leaving his\n",
      "securities with his broker as collateral for the loan that paid for them. The\n",
      "buyer again gets the full benefit of any increase in value—the price of the\n",
      "securities goes up, but the loan that bought them does not. In the stock market\n",
      "the speculative buyer also gets the earnings of the securities he purchased.\n",
      "However, in the days of this history the earnings were almost invariably less\n",
      "than the interest that was paid on the loan. Often they were much less. Yields\n",
      "on securities regularly ranged from nothing to 1 or 2 per cent. Interest on the\n",
      "loans that carried them was often 8, 10, or more per cent. The speculator was\n",
      "willing to pay to divest himself of all of the usufructs of security ownership\n",
      "except the chance for a capital gain.The machinery by which Wall Street\n",
      "separates the opportunity to speculate from the unwanted returns and burdens of\n",
      "ownership is ingenious, precise, and almost beautiful. Banks supply funds to\n",
      "brokers, brokers to customers, and the collateral goes back to banks in a smooth\n",
      "and all but automatic flow. Margins—the cash which the speculator must supply in\n",
      "addition to the securities to protect the loan and which he must augment if the\n",
      "value of the collateral securities should fall and so lower the protection they\n",
      "provide—are effortlessly calculated and watched. The interest rate moves quickly\n",
      "and easily to keep the supply of funds adjusted to the demand. Wall Street,\n",
      "however, has never been able to express its pride in these arrangements. They\n",
      "are admirable and even wonderful only in relation to the purpose they serve. The\n",
      "purpose is to accommodate the speculator and facilitate speculation. But the\n",
      "purposes cannot be admitted. If Wall Street confessed this purpose, many\n",
      "thousands of moral men and women would have no choice but to condemn it for\n",
      "nurturing an evil thing and call for reform. Margin trading must be defended not\n",
      "on the grounds that it efficiently and ingeniously assists the speculator, but\n",
      "that it encourages the extra trading which changes a thin and anemic market into\n",
      "a thick and healthy one. At best this is a dull by-product and a dubious one.\n",
      "Wall Street, in these matters, is like a lovely and accomplished woman who must\n",
      "wear black cotton stockings, heavy woolen underwear, and parade her knowledge as\n",
      "a cook because, unhappily, her supreme accomplishment is as a harlot. However,\n",
      "even the most circumspect friend of the market would concede that the volume of\n",
      "brokers' loans—of loans collateraled by the securities purchased on margin—is a\n",
      "good index of the volume of speculation. Measured by this index, the amount of\n",
      "speculation was rising very fast in 1928. Early in the twenties the volume of\n",
      "brokers' loans—because of their liquidity they are often referred to as call\n",
      "loans or loans in the call market—varied from a billion to a billion and a half\n",
      "dollars. By early 1926 they had increased to two and a half billions and\n",
      "remained at about that level for most of the year. During 1927 there was another\n",
      "increase of about a billion dollars, and at the end of the year they reached\n",
      "$3,480,780,000. This was an incredible sum, but it was only the beginning. In\n",
      "the two dull winter months of 1928 there was a small decline and then expansion\n",
      "began in earnest. Brokers' loans reached four billion on the first of June 1928,\n",
      "five billion on the first of November, and by the end of the year they were well\n",
      "along to six billion.19 Never had there been anything like it before. People\n",
      "were swarming to buy stocks on margin—in other words, to have the increase in\n",
      "price without the costs of ownership. This cost was being assumed, in the first\n",
      "instance, by the New York banks, but they, in turn, were rapidly becoming the\n",
      "agents for lenders the country over and even the world around. There is no\n",
      "mystery as to why so many wished to lend so much in New York. One of the\n",
      "paradoxes of speculation in securities is that the loans that underwrite it are\n",
      "among the safest of all investments. They are protected by stocks which under\n",
      "all ordinary circumstances are instantly salable, and by a cash margin as well.\n",
      "The money, as noted, can be retrieved on demand. At the beginning of 1928 this\n",
      "admirably liquid and exceptionally secure outlet for non-risk capital was paying\n",
      "around 5 per cent. While 5 per cent is an excellent gilt-edged return, the rate\n",
      "rose steadily through 1928, and during the last week of the year it reached 12\n",
      "per cent. This was still with complete safety.In Montreal, London, Shanghai, and\n",
      "Hong Kong there was talk of these rates. Everywhere men of means told themselves\n",
      "that 12 per cent was 12 per cent. A great river of gold began to converge on\n",
      "Wall Street, all of it to help Americans hold common stock on margin.\n",
      "Corporations also found these rates attractive. At 12 per cent Wall Street might\n",
      "even provide a more profitable use for the working capital of a company than\n",
      "additional production. A few firms made this decision: instead of trying to\n",
      "produce goods with its manifold headaches and inconveniences, they confined\n",
      "themselves to financing speculation. Many more companies started lending their\n",
      "surplus funds on Wall Street.There were still better ways of making money. In\n",
      "principle, New York banks could borrow money from the Federal Reserve Bank for 5\n",
      "per cent and re-lend it in the call market for 12. In practice they did. This\n",
      "was, possibly, the most profitable arbitrage operation of all time.\n",
      "VIIIHowever, there were many ways of making money in 1928. Never had there been\n",
      "a better time to get rich, and people knew it. 1928, indeed, was the last year\n",
      "in which Americans were buoyant, uninhibited, and utterly happy. It wasn't that\n",
      "1928 was too good to last; it was only that it didn't last.In the January issue\n",
      "of World's Work, Will Payne, after reflecting on the wonders of the year just\n",
      "over, went on to explain the difference between a gambler and an investor. A\n",
      "gambler, he pointed out, wins only because someone else loses. Where it is\n",
      "investment all gain. One investor, he explained, buys General Motors at $100,\n",
      "sells it to another at $150, who sells it to a third at $200. Everyone makes\n",
      "money. As Walter Bagehot once observed: \"All people are most credulous when they\n",
      "are most happy.\"20          CHAPTER IISomething Should Be Done?PURELY IN\n",
      "RETROSPECT it is easy to see how 1929 was destined to be a year to remember.\n",
      "This was not because Mr. Hoover was soon to become President and had inimical\n",
      "intentions toward the market. Those intentions developed at least partly in\n",
      "retrospect. Nor was it because men of wisdom could tell that a depression was\n",
      "overdue. No one, wise or unwise, knew or now knows when depressions are due or\n",
      "overdue.Rather, it was simply that a roaring boom was in progress in the stock\n",
      "market and, like all booms, it had to end. On the first of January of 1929, as a\n",
      "simple matter of probability, it was most likely that the boom would end before\n",
      "the year was out, with a diminishing chance that it would end in any given year\n",
      "thereafter. When prices stopped rising—when the supply of people who were buying\n",
      "for an increase was exhausted—then ownership on margin would become meaningless\n",
      "and everyone would want to sell. The market wouldn't level out; it would fall\n",
      "precipitately.All this being so, the position of the people who had at least\n",
      "nominal responsibility for what was going on was a complex one. One of the\n",
      "oldest puzzles of politics is who is to regulate the regulators. But an equally\n",
      "baffling problem, which has never received the attention it deserves, is who is\n",
      "to make wise those who are required to have wisdom.Some of those in positions of\n",
      "authority wanted the boom to continue. They were making money out of it, and\n",
      "they may have had an intimation of the personal disaster which awaited them when\n",
      "the boom came to an end. But there were also some who saw, however dimly, that a\n",
      "wild speculation was in progress and that something should be done. For these\n",
      "people, however, every proposal to act raised the same intractable problem. The\n",
      "consequences of successful action seemed almost as terrible as the consequences\n",
      "of inaction, and they could be more horrible for those who took the action.A\n",
      "bubble can easily be punctured. But to incise it with a needle so that it\n",
      "subsides gradually is a task of no small delicacy. Among those who sensed what\n",
      "was happening in early 1929, there was some hope but no confidence that the boom\n",
      "could be made to subside. The real choice was between an immediate and\n",
      "deliberately engineered collapse and a more serious disaster later on. Someone\n",
      "would certainly be blamed for the ultimate collapse when it came. There was no\n",
      "question whatever as to who would be blamed should the boom be deliberately\n",
      "deflated. (For nearly a decade the Federal Reserve authorities had been denying\n",
      "their responsibility for the deflation of 1920–21.) The eventual disaster also\n",
      "had the inestimable advantage of allowing a few more days, weeks, or months of\n",
      "life. One may doubt if at any time in early 1929 the problem was ever framed in\n",
      "terms of quite such stark alternatives. But however disguised or evaded, these\n",
      "were the choices which haunted every serious conference on what to do about the\n",
      "market.         IIThe men who had responsibility for these ineluctable choices\n",
      "were the President of the United States, the Secretary of the Treasury, the\n",
      "Federal Reserve Board in Washington, and the Governor and Directors of the\n",
      "Federal Reserve Bank of New York. As the most powerful of the Federal Reserve\n",
      "Banks, and the one with the market at its doorstep, the New York bank both had\n",
      "and assumed responsibilities which were not accepted by the other eleven banks\n",
      "of the system. President Coolidge neither knew nor cared what was going on. A\n",
      "few days before leaving office in 1929, he cheerily observed that things were\n",
      "\"absolutely sound\" and that stocks were \"cheap at current prices.\" 1 In earlier\n",
      "years, whenever warned that speculation was getting out of hand, he had\n",
      "comforted himself with the thought that this was the primary responsibility of\n",
      "the Federal Reserve Board.2 The Board was a semi-autonomous body precisely\n",
      "because Congress wanted to protect it from excessive political interference by\n",
      "the Executive.However tender his scruples, President Coolidge could have acted\n",
      "through his Secretary of the Treasury, who served, ex-officio, as a member of\n",
      "the Federal Reserve Board. The Secretary also had the primary responsibility for\n",
      "economic and especially for financial policy. But on this as on other matters of\n",
      "economic policy, the incumbent, Andrew W. Mellon, was a passionate advocate of\n",
      "inaction. The responsibility thus passed to the Federal Reserve Board and the\n",
      "Federal Reserve Banks.The regulation of economic activity is without doubt the\n",
      "most inelegant and unrewarding of public endeavors. Almost everyone is opposed\n",
      "to it in principle; its justification always relies on the unprepossessing case\n",
      "for the lesser evil. Regulation originates in raucous debate in Congress in\n",
      "which the naked interests of pressure groups may at times involve an exposure\n",
      "bordering on the obscene. Promulgation and enforcement of rules and regulations\n",
      "is by grinding bureaucracies which are ceaselessly buffeted by criticism. In\n",
      "recent times it has become obligatory for the regulators at every opportunity to\n",
      "confess their inadequacy, which in any case is all too evident. The great\n",
      "exception to this dreary story is the regulatory activity of the central\n",
      "bank—with us, the Federal Reserve System. Here is regulation of a seemly and\n",
      "becoming sort. No one apologizes for it; men of impeccable conservatism would\n",
      "rise to espouse such regulation were they called upon to do so, which they\n",
      "almost never are. This regulation is not the work of thousands of clerks,\n",
      "statisticians, hearing officers, lawyers, and lesser beings in a teeming office\n",
      "building on the Mall. Rather it emerges in the measured and orderly discussion\n",
      "of men of quiet and dignified mien, each at his accustomed place around a\n",
      "handsome table in a richly paneled and richly draperied room. These men do not\n",
      "issue orders; at most they suggest. Chiefly they move interest rates, buy or\n",
      "sell securities and, in doing so, nudge the economy here and restrain it there.\n",
      "Because the meanings of their actions are not understood by the great majority\n",
      "of the people, they can reasonably be assumed to have superior wisdom. Their\n",
      "actions will on occasion be criticized. More often they will be scrutinized for\n",
      "hidden meaning.Such is the mystique of central banking. Such was the awe-\n",
      "inspiring role in 1929 of the Federal Reserve Board in Washington, the policy-\n",
      "making body which guided and directed the twelve Federal Reserve banks. However,\n",
      "there was a jarring difficulty. The Federal Reserve Board in those times was a\n",
      "body of startling incompetence.For several years, until late in 1927, the\n",
      "Chairman and guiding genius presumptive, was one Daniel R. Crissinger. He had\n",
      "been trained for his task by serving as General Counsel of the Marion Steam\n",
      "Shovel Company of Marion, Ohio. There is no indication that he was an apt\n",
      "student. However, his background seemed satisfactory to another Marion boy,\n",
      "Warren G. Harding, who had brought him to Washington, where he was regarded as a\n",
      "hack politician from Ohio. In 1927 Crissinger was replaced by Roy A. Young, who\n",
      "for eight years had been Governor of the Minneapolis Federal Reserve Bank.\n",
      "Young, a more substantial figure, was undoubtedly aware of what was going on.\n",
      "However, he was a man of caution who sought no fame as a martyr to the broken\n",
      "boom. His colleagues were among the more commonplace of Harding-Coolidge\n",
      "appointees. With one exception—the erstwhile college professor, Adolph C.\n",
      "Miller—they have been conservatively described by Herbert Hoover as\n",
      "\"mediocrities.\"3 The New York Federal Reserve Bank was under more vigorous\n",
      "leadership. For several years, until 1928, its governor had been Benjamin\n",
      "Strong, the first American since Nicholas Biddle to make an important reputation\n",
      "as a central banker. Strong's views were regarded throughout the System with\n",
      "only little less awe than the gold standard. However, in the view of Herbert\n",
      "Hoover—and in this instance Hoover's views are widely shared—Strong, so far from\n",
      "being concerned about the inflation, was the man most responsible for it. It was\n",
      "he who took the lead in 1927 in easing money rates to help the hard-pressed\n",
      "Europeans. For this Mr. Hoover later called him \"a mental annex to Europe.\"4This\n",
      "is unfair. Governor Strong's action was entirely reasonable in the circumstances\n",
      "and, as noted in the last chapter, it takes more to start a speculation than a\n",
      "general ability to borrow money. Still, the New York Federal Reserve Bank, under\n",
      "Governor Strong's leadership may not have been sufficiently perturbed by the\n",
      "speculation a block or two away. Nor was it after Governor Strong died in\n",
      "October 1928 and was replaced by George L. Harrison. A reason, no doubt, was the\n",
      "reassurance provided by people in high places who were themselves speculating\n",
      "heavily. One such was Charles E. Mitchell, the Chairman of the Board of the\n",
      "National City Bank who, on January 1, 1929, became a class A director of the\n",
      "Federal Reserve Bank of New York. The end of the boom would mean the end of\n",
      "Mitchell. He was not a man to expedite his own demise.         IIIIn the\n",
      "accepted history of these times, the Federal Reserve authorities are held to be\n",
      "not so much unaware or unwilling as impotent. They would have liked to stop the\n",
      "boom, but they lacked the means. This puts far too elaborate a face on matters.\n",
      "And it largely disguises the real nature of the dilemma which the authorities\n",
      "faced.The classic instruments of control were indeed largely useless. These, as\n",
      "almost every college sophomore knows, are two: open market operations and the\n",
      "manipulation of the rediscount rate. Open market sales of government securities5\n",
      "by the Federal Reserve bring to the vaults of the Reserve Banks the cash which\n",
      "is paid for the securities. There it remains sterile and harmless. Had it stayed\n",
      "in the commercial banks it would have been loaned to the public in multiple\n",
      "volume and particularly in those days to people who were buying common stocks.\n",
      "If such a policy is to succeed, the Federal Reserve System rather obviously must\n",
      "have securities to sell. One of the inestimable blessings of the years of\n",
      "depression, war, and deficit-financing since 1930 is a spacious inventory of\n",
      "government debt in the Reserve Banks. In 1929 the Banks were not so well\n",
      "endowed. At the beginning of 1928, holdings were $617 million. During the first\n",
      "half of the year there were heavy sales as part of an effort to dry up the\n",
      "supply of funds that was feeding the market. Although sales were discontinued in\n",
      "the latter part of the year in the highly erroneous belief that the policy had\n",
      "succeeded and that the boom was under control, they couldn't have been continued\n",
      "much longer in any case. By the end of 1928 the inventory of government\n",
      "securities of the Federal Reserve System amounted to only $228 million. Had\n",
      "these all been dropped into the market, they might possibly have had some\n",
      "effect. But the Board was not given to any such drastic behavior which,\n",
      "incidentally, would also have largely denuded the Reserve Banks of earning\n",
      "assets. Sales were made a few millions at a time in the early months of 1929,\n",
      "but the effect was inconsequential. Moreover, even in following this feeble\n",
      "policy the Board worried lest, in denying funds to the stock market, it might\n",
      "put a pinch on \"legitimate\" business. The Reserve Banks continued to buy\n",
      "acceptances—the security that emerges in the course of financing ordinary non-\n",
      "speculative trade—and, relieved of the need to carry this paper, the commercial\n",
      "banks happily loaned more money in the stock market.The other instrument of\n",
      "Federal Reserve policy was the rediscount rate. This is the rate at which member\n",
      "commercial banks borrow from the Reserve Banks of their district so that they\n",
      "may accommodate more borrowers than their own resources permit. In January 1929,\n",
      "the rediscount rate at the New York Federal Reserve Bank was 5 per cent. The\n",
      "rate on brokers' loans ranged from 6 to 12 per cent. Only a drastic increase\n",
      "would have made it unprofitable for a bank to borrow at the Federal Reserve in\n",
      "order to lend the proceeds, directly or indirectly, in the stock market. Apart\n",
      "from the general aversion to drastic action, such an increase would also have\n",
      "raised rates to ordinary businessmen, consumers, and farmers. In fact, higher\n",
      "interest rates would have been distressing to everyone but the speculator. A man\n",
      "who paid, say, an average of 10 per cent to carry his holdings of Radio through\n",
      "1928 would not have been either deterred or much disturbed had the rate been\n",
      "twice that high. During the same year, he made 500 per cent on the appreciation\n",
      "in the value of his investment. On February 14, 1929, the New York Federal\n",
      "Reserve Bank proposed that the rediscount rate be raised from 5 to 6 per cent to\n",
      "check speculation. The Federal Reserve Board in Washington thought this a\n",
      "meaningless gesture which would only increase rates to business borrowers. A\n",
      "long controversy ensued in which President Hoover sided with the Board against\n",
      "the Bank. The rate was not increased until late in the summer.There was another\n",
      "circumstance which gave the Reserve authorities an admirable excuse for\n",
      "inaction. That was the previously noted flow of funds from corporations and\n",
      "individuals to the market. During 1929, Standard Oil of New Jersey contributed a\n",
      "daily average of $69 million to the call market; Electric Bond and Share\n",
      "averaged over $100 million.6 A few corporations—Cities Service was one—even sold\n",
      "securities and loaned the proceeds in the stock market.7 By early 1929, loans\n",
      "from these non-banking sources were approximately equal to those from the banks.\n",
      "Later they became much greater. The Federal Reserve authorities took for granted\n",
      "that they had no influence whatever over this supply of funds.         IVIn\n",
      "fact, the Federal Reserve was helpless only because it wanted to be. Had it been\n",
      "determined to do something, it could for example have asked Congress for\n",
      "authority to halt trading on margin by granting the Board the power to set\n",
      "margin requirements. Margins were not low in 1929; a residue of caution had\n",
      "caused most brokers to require customers to put up in cash 45 to 50 per cent of\n",
      "the value of the stocks they were buying. However, this was all the cash\n",
      "numerous of their customers had. An increase in margins to, say, 75 per cent in\n",
      "January 1929, or even a serious proposal to do so, would have caused many small\n",
      "speculators and quite a few big ones to sell. The boom would have come to a\n",
      "sudden and perhaps spectacular end. (The power to fix margin requirements was\n",
      "eventually given to the Federal Reserve Board by the Securities Exchange Act in\n",
      "1934, a year in which the danger of a revival of speculation about equaled that\n",
      "of a renascence of prohibition.)Actually, not even new legislation, or the\n",
      "threat of it, was needed. In 1929, a robust denunciation of speculators and\n",
      "speculation by someone in high authority and a warning that the market was too\n",
      "high would almost certainly have broken the spell. It would have brought some\n",
      "people back from the world of make-believe. Those who were planning to stay in\n",
      "the market as long as possible but still get out (or go short) in time would\n",
      "have got out or gone short. Their occupational nervousness could readily have\n",
      "been translated into an acute desire to sell. Once the selling started, some\n",
      "more vigorously voiced pessimism could easily have kept it going. The very\n",
      "effectiveness of such a measure was the problem. Of all the weapons in the\n",
      "Federal Reserve arsenal, words were the most unpredictable in their\n",
      "consequences. Their effect might be sudden and terrible. Moreover, these\n",
      "consequences could be attributed with the greatest of precision to the person or\n",
      "persons who uttered the words. Retribution would follow. To the more cautious of\n",
      "the Federal Reserve officials in the early part of 1929 silence seemed literally\n",
      "golden.Yet the boom was continuing. In January the Times industrials gained 30\n",
      "points, more than during the post-election spree in November. Brokers' loans\n",
      "went up a whopping $260 million; on five different days, three of them as the\n",
      "market got off to a boiling start right after New Year's, trading on the New\n",
      "York Stock Exchange exceeded the magic five-million mark. Effective action would\n",
      "be disastrous; yet some action seemed unavoidable. Finally the Board decided to\n",
      "write a letter and issue a press release. It could do no less.On February 2, it\n",
      "addressed the individual Reserve Banks as follows: A member [commercial bank] is\n",
      "not within its reasonable claims for re-discount facilities at its reserve bank\n",
      "when it borrows either for the purpose of making speculative loans or for the\n",
      "purpose of maintaining speculative loans. The board has no disposition to assume\n",
      "authority to interfere with the loan practices of member banks, so long as they\n",
      "do not involve the Federal reserve banks. It has, however, a grave\n",
      "responsibility whenever there is evidence that member banks are maintaining\n",
      "speculative security loans with the aid of Federal reserve credit.8 On February\n",
      "7, in what may be an even finer example of fiduciary prose—connoisseurs will\n",
      "wish to read it backward as well as forward—the Board warned the public:  When\n",
      "[the Board] finds that conditions are arising which obstruct the Federal reserve\n",
      "banks in the effective discharge of their functions of so managing the credit\n",
      "facilities of the Federal reserve system as to accommodate commerce and\n",
      "business, it is its duty to inquire into them and to take such measures as may\n",
      "be deemed suitable and effective in the circumstances to correct them; which, in\n",
      "the immediate situation, means to restrain the use, either directly or\n",
      "indirectly, of Federal Reserve facilities in aid of the growth of speculative\n",
      "credit.9 Almost simultaneously with this warning came the news that the Bank of\n",
      "England was raising the bank rate from 4)4 to 5)4 per cent in an effort to\n",
      "diminish the flow of British funds to the new Golconda. The result was a sharp\n",
      "break in the market. On February 7, in a five-million share day, the Times\n",
      "industrials dropped 11 points, with a further drop on the day following.\n",
      "Thereafter the market recovered, but, for February as a whole, there was no\n",
      "appreciable net gain. Economists have long had a phrase for this action—it was\n",
      "called the Federal Reserve's effort at \"moral suasion.\" Since the market was\n",
      "only temporarily checked, there has been ever since virtually complete agreement\n",
      "that moral suasion was a failure.Precisely the opposite conclusion could and\n",
      "probably should have been drawn. It is impossible to imagine a milder, more\n",
      "tentative, more palpably panic-stricken communiqué than that issued by the\n",
      "Board. The statement that the Board had no intention of interfering with loans\n",
      "to support speculation so long as Federal Reserve credit was not involved is\n",
      "especially noteworthy. Clearly the Federal Reserve was less interested in\n",
      "checking speculation than in detaching itself from responsibility for the\n",
      "speculation that was going on. And it will be observed that some anonymous\n",
      "draftsman achieved a wording which indicated that not the present level but only\n",
      "a further growth in speculation would be viewed with alarm. Yet in the then\n",
      "state of nervousness even these almost incredibly feeble words caused a sharp\n",
      "setback.         VThe nervousness of the market and the unsuspected moral\n",
      "authority of the equally nervous men of the Federal Reserve was even better\n",
      "illustrated in March. As the new month approached, Mr. Coolidge made his blithe\n",
      "observation about stocks being cheap and the country being sound. The market\n",
      "surged up in what the papers dubbed \"The Inaugural Market,\" and on March 4, his\n",
      "attitude toward speculators still unknown, Mr. Hoover took over. The market for\n",
      "the next couple of weeks remained strong.Then toward the end of the month\n",
      "disquieting news reached Wall Street. The Federal Reserve Board was meeting\n",
      "daily in Washington. It issued no statements. Newspapermen pressed the members\n",
      "after the sessions and were met with what then, as now, was known as tight-\n",
      "lipped silence. There was not a hint as to what the meetings were about,\n",
      "although everyone knew they concerned the market. The meetings continued day\n",
      "after day, and there was also an unprecedented Saturday session.Soon it was too\n",
      "much. On Monday, March 25, the first market day following the unseemly Saturday\n",
      "meeting, the tension became unbearable. Although, or rather because, Washington\n",
      "was still silent, people began to sell. Speculative favorites—Commercial\n",
      "Solvents, Wright Aero, American Railway Express—dropped 10 or 12 points or more;\n",
      "the Times industrial average was off 9½ points for the day. More important, some\n",
      "banks decided that, in the event of a Federal Reserve crackdown, virtue might\n",
      "have a reward above revenue. They began curtailing their loans in the call\n",
      "market, and the rate on brokers' loans went to 14 per cent. On the next day,\n",
      "Tuesday, March 26, everything was much worse. The Federal Reserve Board was\n",
      "still maintaining its by now demoralizing silence. A wave of fear swept the\n",
      "market. More people decided to sell, and they sold in astonishing volume. An\n",
      "amazing 8,246,740 shares changed hands on the New York Stock Exchange, far above\n",
      "any previous record. Prices seemed to drop vertically. At the low for the day\n",
      "20– and 30-point losses were commonplace. The Times industrials at one time were\n",
      "15 points below the previous day's close. Thousands of speculators, in whose\n",
      "previous experience the market had always risen, now saw for the first time the\n",
      "seamy side of their new way of life. Each new quotation was far below the last\n",
      "one. Moreover, the ticker, unable to cope with the unprecedented volume, was far\n",
      "behind the market. Not only were things bad, but they had almost certainly\n",
      "become worse, yet one couldn't tell from the ticker how much worse. Before the\n",
      "day was out many of these thousands received a peremptory telegram from their\n",
      "brokers—a telegram that was in the starkest of contrast with the encouraging,\n",
      "half-confidential, rich-uncle tone of all previous communications. It asked more\n",
      "margin promptly.Meanwhile the banks continued to trim sails for the storm. It is\n",
      "probable that some professional traders were selling because they foresaw the\n",
      "moment when there would be no money with which to carry stocks on margin. And\n",
      "that moment might be near, for on the morning of March 26, the rate on call\n",
      "money reached 20 per cent, its high for the 1929 boom. March 26, 1929, could\n",
      "have been the end. Money could have remained tight. The authorities might have\n",
      "remained firm in their intention to keep it so. The panic might have continued.\n",
      "Each fall in prices would have forced a new echelon of speculators to sell, and\n",
      "so forced prices down still more. It did not happen, and if any man can be\n",
      "credited with this, the credit belongs to Charles E. Mitchell. The Federal\n",
      "Reserve authorities were ambivalent, but Mitchell was not. He was for the boom.\n",
      "Moreover, his prestige as head of one of the two largest and most influential\n",
      "commercial banks, his reputation as an aggressive and highly successful\n",
      "investment banker, and his position as a director of the New York Federal\n",
      "Reserve Bank meant that he spoke with at least as much authority as anyone in\n",
      "Washington. During the day, as money tightened, rates rose, and the market fell,\n",
      "Mitchell decided to take a hand. He told the press, \"We feel that we have an\n",
      "obligation which is paramount to any Federal Reserve warning, or anything else,\n",
      "to avert any dangerous crisis in the money market.\" The National City, he said,\n",
      "would loan money as necessary to prevent liquidation. It would also (and did)\n",
      "borrow from the New York Federal Reserve Bank to do what the Federal Reserve\n",
      "Board had warned against doing. Disguised only slightly by the prose form of\n",
      "finance, Mitchell issued the Wall Street counterpart of Mayor Hague's famous\n",
      "manifesto, \"I am the law in Jersey City.\"Mitchell's words were like magic. By\n",
      "the end of trading on the 26th money rates had eased, and the market had\n",
      "rallied. The Federal Reserve remained silent, but now its silence was\n",
      "reassuring. It meant that it conceded Mitchell's mastery. The next day the\n",
      "National City regularized its commitment to the boom: it announced that it would\n",
      "insure reasonable interest rates by putting $25 million into the call market—$5\n",
      "million when the rate was 16 per cent, and $5 million additional for each\n",
      "percentage point. In its monthly letter a few days later, the bank justified its\n",
      "position and, incidentally, gave an admirable statement of the dilemma which the\n",
      "Federal Reserve faced. (The National City was in no doubt which horn of the\n",
      "dilemma it preferred.) The letter said: \"The National City Bank fully recognizes\n",
      "the dangers of overspeculation [sic] and endorses the desire of the Federal\n",
      "Reserve authorities to restrain excessive credit expansion for this purpose. At\n",
      "the same time, the bank, business generally, and it may be assumed the Federal\n",
      "Reserve Banks ... wish to avoid a general collapse of the securities markets\n",
      "such as would have a disastrous effect on business.\"10         VIMitchell did\n",
      "not escape criticism. A Senate investigation was mentioned. Senator Carter\n",
      "Class, who as the sponsor of the Federal Reserve legislation had a keen\n",
      "proprietary interest in its operation, said: \"He avows his superior obligation\n",
      "to a frantic stock market over against the obligation of his oath as a director\n",
      "of the New York Federal Reserve Bank ... the Bank should ask for [his] immediate\n",
      "resignation.\" That the Federal Reserve Board never dreamed of doing any such\n",
      "thing may be taken as a further indication that its moral suasion was being\n",
      "administered with a very infirm touch. And the Federal Reserve was criticized\n",
      "even more than Mitchell—even though it could hardly have done less than it did.\n",
      "Arthur Brisbane said judiciously: \"If buying and selling stocks is wrong the\n",
      "government should close the Stock Exchange. If not, the Federal Reserve should\n",
      "mind its own business.\" In a leading article in Barron's, a Mr. Seth Axley was\n",
      "less even-handed: \"For the Federal Reserve Board to deny investors the means of\n",
      "recognizing economies which are now proved, skill which is now learned, and\n",
      "inventions which are almost unbelievable seems to justify doubt whether it is\n",
      "adequately interpreting the times.\"11 Since the principal action which the\n",
      "Federal Reserve had taken against investors had been to hold meetings and\n",
      "maintain silence, this was doubtless a trifle harsh. However, it was positively\n",
      "kind as compared with the words of a young Princeton scholar who emerged at this\n",
      "time as a leading defender of Wall Street.Joseph Stagg Lawrence's book Wall\n",
      "Street and Washington, when it appeared later in the year bearing the\n",
      "distinguished imprimatur of the Princeton University Press, was described by a\n",
      "leading financial journal as a \"breath of fresh air.\" The regulatory concern of\n",
      "the Federal Reserve about Wall Street, Mr. Lawrence said in this remarkable\n",
      "volume, was motivated strictly by bias—a bias \"founded upon a clash of interests\n",
      "and a moral and intellectual antipathy between the wealthy, cultured, and\n",
      "conservative settlements on the seacoast [including Wall Street] and the poverty\n",
      "stricken, illiterate, and radical pioneer communities of the interior.\" 12 The\n",
      "cultured and conservative Mr. Lawrence also had hard words for the defenders of\n",
      "the Federal Reserve in the Senate, including, oddly enough, the seacoast senator\n",
      "from Virginia, Carter Glass. \"It seems incredible that in the year of our Grace\n",
      "1929 a body of presumably intelligent public men should permit fanatical\n",
      "passions and provincial ignorance to find expression in unrestrained virulence.\n",
      "Yet this is precisely what has taken place ... When the Senator from the Old\n",
      "Dominion rose in that chamber of absurdities, sometimes referred to as a\n",
      "deliberative assembly, his remarks were characterized by neither reason nor\n",
      "restraint. Blatant bigotry and turbulent provincialism have joined to condemn an\n",
      "innocent community.\"13 Some hardened Wall Streeters may have been surprised when\n",
      "they realized that the \"innocent community\" meant them.         VIIAfter the\n",
      "defeat by Mitchell in March, the Federal Reserve retired from the field. There\n",
      "continued to be some slight anxiety as to what it might do. In April, William\n",
      "Crapo Durant is supposed to have paid a secret night visit to the White House to\n",
      "wam President Hoover that if the Board were not called off it would precipitate\n",
      "a terrible crash. The President was noncommittal, and Durant is said to have\n",
      "reduced his holdings before leaving on a trip to Europe.14 In June from\n",
      "Princeton Mr. Lawrence said that the Board was still \"doing its utmost to cast\n",
      "the proverbial monkey wrench into the machinery of prosperity.\" He warned the\n",
      "Board that it had \"aroused the enmity of an honest, intelligent, and public-\n",
      "spirited community.\"15 (This also was Wall Street.) But the Board, in fact, had\n",
      "decided to leave that honest, intelligent, and public-spirited community\n",
      "strictly to its own devices. It realized, Governor Young said subsequently, that\n",
      "\"while the hysteria might be somewhat restrained,\" it would have to run its\n",
      "course, and the Reserve Banks could only brace themselves for the \"inevitable\n",
      "collapse.\"16 More accurately, the Federal Reserve authorities had decided not to\n",
      "be responsible for the collapse. In August the Board finally agreed to an\n",
      "increase in the rediscount rate to 6 per cent. The market weakened only for a\n",
      "day. Any conceivable consequence of the action was nullified by a simultaneous\n",
      "easing of the buying rate on acceptances.In fact, from the end of March on, the\n",
      "market had nothing further to fear from authority. President Hoover did ask\n",
      "Henry M. Robinson, a Los Angeles banker, to proceed as his emissary to New York\n",
      "and talk to the bankers there about the boom. According to Mr. Hoover, Robinson\n",
      "was assured that things were sound.17 Richard Whitney, the Vice-President of the\n",
      "Exchange, was also summoned to the White House and told that something should be\n",
      "done about speculation. Nothing was done, and Mr. Hoover was able to find some\n",
      "solace in the thought that primary responsibility for regulating the stock\n",
      "exchange rested with the Governor of New York, Franklin D. Roosevelt.18\n",
      "Roosevelt, too, was following a laissez-faire policy, at least on the matter of\n",
      "the stock market. A firm of investment counselors in Boston, called McNeel's\n",
      "Financial Service, describing itself with a deft Back Bay touch as \"An\n",
      "Aristocracy of Successful Investors,\" advertised a new guide to investment. The\n",
      "headline read: \"He made $70,000 after reading Beating the Stock Market\" No doubt\n",
      "whoever it was did. He might have made it without reading the volume or without\n",
      "being able to read. For now, free at last from all threat of government reaction\n",
      "or retribution, the market sailed off into the wild blue yonder. Especially\n",
      "after June 1 all hesitation disappeared. Never before or since have so many\n",
      "become so wondrously, so effortlessly, and so quickly rich. Perhaps Messrs.\n",
      "Hoover and Mellon, and the Federal Reserve were right in keeping their hands\n",
      "off. Perhaps it was worth being poor for a long time to be so rich for just a\n",
      "little while.         CHAPTER IIIIn Goldman, Sachs We TrustTHE RECONDITE\n",
      "PROBLEMS of Federal Reserve policy were not the only questions that were\n",
      "agitating Wall Street intellectuals in the early months of 1929. There was worry\n",
      "that the country might be running out of common stocks. One reason prices of\n",
      "stocks were so high, it was explained, was that there weren't enough to go\n",
      "around, and, accordingly, they had acquired a \"scarcity value.\" Some issues, it\n",
      "was said, were becoming so desirable that they would soon be taken out of the\n",
      "market and would not reappear at any price.If, indeed, common stocks were\n",
      "becoming scarce it was in spite of as extraordinary a response of supply to\n",
      "demand as any in the history of that well-worn relationship. Without doubt, the\n",
      "most striking feature of the financial era which ended in the autumn of 1929 was\n",
      "the desire of people to buy securities and the effect of this on values. But the\n",
      "increase in the number of securities to buy was hardly less striking. And the\n",
      "ingenuity and zeal with which companies were devised in which securities might\n",
      "be sold was as remarkable as anything.Not all of the increase in the volume of\n",
      "securities in 1928 and 1929 was for the sole purpose of accommodating the\n",
      "speculator. It was a good time to raise money for general corporate purposes.\n",
      "Investors would supply capital with enthusiasm and without tedious questions.\n",
      "(Seaboard Air Line was a speculative favorite of the period in part because many\n",
      "supposed it to be an aviation stock with growth possibilities.) In these years\n",
      "of prosperity men with a vision of still greater prosperity stretching on and on\n",
      "and forever, naturally saw the importance of being well provided with plant and\n",
      "working capital. This was no time to be niggardly. Also, it was an age of\n",
      "consolidation, and each new merger required, inevitably, some new capital and a\n",
      "new issue of securities to pay for it. A word must be said about the merger\n",
      "movement of the twenties.It was not the first such movement but, in many\n",
      "respects, it was the first of its kind. Just before and just after the turn of\n",
      "the century in industry after industry, small companies were combined into large\n",
      "ones. The United States Steel Corporation, International Harvester,\n",
      "International Nickel, American Tobacco, and numerous other of the great\n",
      "corporations trace to this period. In these cases the firms which were combined\n",
      "produced the same or related products for the same national market. The primary\n",
      "motivation in all but the rarest cases was to reduce, eliminate, or regularize\n",
      "competition. Each of the new giants dominated an industry, and henceforth\n",
      "exercised measurable control over prices and production, and perhaps also over\n",
      "investment and the rate of technological innovation.A few such mergers occurred\n",
      "in the twenties. Mostly, however, the mergers of this period brought together\n",
      "not firms in competition with each other but firms doing the same thing in\n",
      "different communities. Local electric, gas, water, bus, and milk companies were\n",
      "united in great regional or national systems. The purpose was not to eliminate\n",
      "competition, but rather the incompetence, somnambulance, naïveté, or even the\n",
      "unwarranted integrity of local managements. In the twenties, a man in downtown\n",
      "New York or Chicago could take unabashed pride in the fact that he was a\n",
      "financial genius. The local owners and managers were not. There was no false\n",
      "modesty when it came to citing the advantages of displacing yokels with a\n",
      "central management of decent sophistication. In the case of utilities the\n",
      "instrument for accomplishing this centralization of management and control was\n",
      "the holding company. These bought control of the operating companies. On\n",
      "occasion they bought control of other holding companies which controlled yet\n",
      "other holding companies, which in turn, directly or indirectly through yet other\n",
      "holding companies, controlled the operating companies. Everywhere local power,\n",
      "gas, and water companies passed into the possession of a holding-company\n",
      "system.Food retailing, variety stores, department stores, and motion picture\n",
      "theatres showed a similar, although not precisely identical, development. Here,\n",
      "too, local ownership gave way to central direction and control. The instrument\n",
      "of this centralization, however, was not the holding company but the corporate\n",
      "chain. These, more often than not, instead of taking over existing businesses,\n",
      "established new outlets.The holding companies issued securities in order to buy\n",
      "operating properties, and the chains issued securities in order to build new\n",
      "stores and theatres. While in the years before 1929 the burgeoning utility\n",
      "systems—Associated Gas and Electric, Commonwealth and Southern, and the Insull\n",
      "companies—attracted great attention, the chains were at least as symbolic of the\n",
      "era. Montgomery Ward was one of the prime speculative favorites of the period;\n",
      "it owed its eminence to the fact that it was a chain and thus had a particularly\n",
      "bright future. The same was true of Woolworth, American Stores, and others.\n",
      "Interest in branch and chain banking was also strong, and it was widely felt\n",
      "that state and federal laws were an archaic barrier to a consolidation which\n",
      "would knit the small-town and small-city banks into a few regional and national\n",
      "systems. Various arrangements for defeating the intent of the law, most notably\n",
      "bank holding companies, were highly regarded.Inevitably promoters organized some\n",
      "new companies merely to capitalize on the public interest in industries with a\n",
      "new and wide horizon and provide securities to sell. Radio and aviation stocks\n",
      "were believed to have a particularly satisfactory prospect, and companies were\n",
      "formed which never had more than a prospect. In September 1929, an advertisement\n",
      "in the Times called attention to the impending arrival of television and said\n",
      "with considerable prescience that the \"commercial possibilities of this new art\n",
      "defy imagination.\" The ad opined, somewhat less presciently, that sets would be\n",
      "in use in homes that fall. However, in the main, the market boom of 1929 was\n",
      "rooted directly or indirectly in existing industries and enterprises. New and\n",
      "fanciful issues for new and fanciful purposes, ordinarily so important in times\n",
      "of speculation, played a relatively small part. No significant amount of stock\n",
      "was sold in companies \"To make Salt Water Fresh—For building of Hospitals for\n",
      "Bastard Children—For building of Ships against Pirates—For importing a Number of\n",
      "large Jack Asses from Spain,\" or even \"For a Wheel of Perpetual Motion,\" to cite\n",
      "a representative list of promotions at the time of the South Sea Bubble.1\n",
      "IIThe most notable piece of speculative architecture of the late twenties, and\n",
      "the one by which, more than any other device, the public demand for common\n",
      "stocks was satisfied, was the investment trust or company. The investment trust\n",
      "did not promote new enterprises or enlarge old ones. It merely arranged that\n",
      "people could own stock in old companies through the medium of new ones. Even in\n",
      "the United States, in the twenties, there were limits to the amount of real\n",
      "capital which existing enterprises could use or new ones could be created to\n",
      "employ. The virtue of the investment trust was that it brought about an almost\n",
      "complete divorce of the volume of corporate securities outstanding from the\n",
      "volume of corporate assets in existence. The former could be twice, thrice, or\n",
      "any multiple of the latter. The volume of underwriting business and of\n",
      "securities available for trading on the exchanges all expanded accordingly. So\n",
      "did the securities to own, for the investment trusts sold more securities than\n",
      "they bought. The difference went into the call market, real estate, or the\n",
      "pockets of the promoters. It is hard to imagine an invention better suited to\n",
      "the time or one better designed to eliminate the anxiety about the possible\n",
      "shortage of common stocks. The idea of the investment trust is an old one,\n",
      "although, oddly enough, it came late to the United States. Since the eighteen-\n",
      "eighties in England and Scotland, investors, mostly smaller ones, had pooled\n",
      "their resources by buying stock in an investment company. The latter, in turn,\n",
      "invested the funds so secured. A typical trust held securities in from five\n",
      "hundred to a thousand operating companies. As a result, the man with a few\n",
      "pounds, or even a few hundred, was able to spread his risk far more widely than\n",
      "were he himself to invest. And the management of the trusts could be expected to\n",
      "have a far better knowledge of companies and prospects in Singapore, Madras,\n",
      "Capetown, and the Argentine, places to which British funds regularly found their\n",
      "way, than the widow in Bristol or the doctor in Glasgow. The smaller risk and\n",
      "better information well justified the modest compensation of those who managed\n",
      "the enterprise. Despite some early misadventures, the investment trusts soon\n",
      "became an established part of the British scene. Before 1921 in the United\n",
      "States only a few small companies existed for the primary purpose of investing\n",
      "in the securities of other companies.2 In that year, interest in investment\n",
      "trusts began to develop, partly as the result of a number of newspaper and\n",
      "magazine articles describing the English and Scottish trusts. The United States,\n",
      "it was pointed out, had not been keeping abreast of the times; other countries\n",
      "were excelling us in fiduciary innovation. Soon, however, we began to catch up.\n",
      "More trusts were organized, and by the beginning of 1927 an estimated 160 were\n",
      "in existence. Another 140 were formed during that year.3The managers of the\n",
      "British trusts normally enjoy the greatest of discretion in investing the funds\n",
      "placed at their disposal. At first the American promoters were wary of asking\n",
      "for such a vote of confidence. Many of the early trusts were trusts—the investor\n",
      "bought an interest in a specified assortment of securities which were then\n",
      "deposited with a trust company. At the least the promoters committed themselves\n",
      "to a rigorous set of rules on the kinds of securities to be purchased and the\n",
      "way they were to be held and managed. But as the twenties wore along, such\n",
      "niceties disappeared. The investment trust became, in fact, an investment\n",
      "corporation.4 It sold its securities to the public—sometimes just common stock,\n",
      "more often common and preferred stock, debenture and mortgage bonds—and the\n",
      "proceeds were then invested as the management saw fit. Any possible tendency of\n",
      "the common stockholder to interfere with the management was prevented by selling\n",
      "him non-voting stock or having him assign his voting rights to a management-\n",
      "controlled voting trust.For a long time the New York Stock Exchange looked with\n",
      "suspicion on the investment trusts; only in 1929 was fisting permitted. Even\n",
      "then the Committee on the Stock List required an investment trust to post with\n",
      "the Exchange the book and market value of the securities held at the time of\n",
      "listing and once a year thereafter to provide an inventory of its holdings. This\n",
      "provision confined the listing of most of the investment trusts to the Curb,\n",
      "Boston, Chicago, or other road company exchanges. Apart from its convenience,\n",
      "this refusal to disclose holdings was thought to be a sensible precaution.\n",
      "Confidence in the investment judgment of the managers of the trusts was very\n",
      "high. To reveal the stocks they were selecting might, it was said, set off a\n",
      "dangerous boom in the securities they favored. Historians have told with wonder\n",
      "of one of the promotions at the time of the South Sea Bubble. It was \"For an\n",
      "Undertaking which shall in due time be revealed.\" The stock is said to have sold\n",
      "exceedingly well. As promotions the investment trusts were, on the record, more\n",
      "wonderful. They were undertakings the nature of which was never to be revealed,\n",
      "and their stock also sold exceedingly well.         IIIDuring 1928 an estimated\n",
      "186 investment trusts were organized; by the early months of 1929 they were\n",
      "being promoted at the rate of approximately one each business day, and a total\n",
      "of 265 made their appearance during the course of the year. In 1927 the trusts\n",
      "sold to the public about $400,000,000 worth of securities; in 1929 they marketed\n",
      "an estimated three billions worth. This was at least a third of all the new\n",
      "capital issues in that year; by the autumn of 1929 the total assets of the\n",
      "investment trusts were estimated to exceed eight billions of dollars. They had\n",
      "increased approximately elevenfold since the beginning of 1927.5 The\n",
      "parthenogenesis of an investment trust differed from that of an ordinary\n",
      "corporation. In nearly all cases it was sponsored by another company, and by\n",
      "1929 a surprising number of different kinds of concerns were bringing the trusts\n",
      "into being. Investment banking houses, commercial banks, brokerage firms,\n",
      "securities dealers, and, most important, other investment trusts were busy\n",
      "giving birth to new trusts. The sponsors ranged in dignity from the House of\n",
      "Morgan, sponsor of the United and Alleghany Corporations, down to one Chauncey\n",
      "D. Parker, the head of a fiscally perilous investment banking firm in Boston,\n",
      "who organized three investment trusts in 1929 and sold $25,000,000 worth of\n",
      "securities to an eager public. Chauncey then lost most of the proceeds and\n",
      "lapsed into bankruptcy.6Sponsorship of a trust was not without its rewards. The\n",
      "sponsoring firm normally executed a management contract with its offspring.\n",
      "Under the usual terms, the sponsor ran the investment trust, invested its funds,\n",
      "and received a fee based on a percentage of capital or earnings. Were the\n",
      "sponsor a stock exchange firm, it also received commissions on the purchase and\n",
      "sale of securities for its trust. Many of the sponsors were investment banking\n",
      "firms, which meant, in effect, that the firm was manufacturing securities it\n",
      "could then bring to market. This was an excellent way of insuring an adequate\n",
      "supply of business. The enthusiasm with which the public sought to buy\n",
      "investment trust securities brought the greatest rewards of all. Almost\n",
      "invariably people were willing to pay a sizable premium over the offering price.\n",
      "The sponsoring firm (or its promoters) received allotments of stock or warrants\n",
      "which entitled them to stock at the offering price. These they were then able to\n",
      "sell at once at a profit. Thus one of the enterprises of the Mr. Chauncey D.\n",
      "Parker just mentioned—a company with the resounding name of Seaboard Utilities\n",
      "Shares Corporation—issued 1,600,000 shares of common stock on which the company\n",
      "netted $10.32 a share. That, however, was not the price paid by the public. It\n",
      "was the price at which the stock was issued to Parker and his colleagues. They\n",
      "in turn sold their shares to the public at from $11 to $18.25 and split the\n",
      "profit with the dealers who marketed the securities.7Operations of this sort\n",
      "were not confined to the lowly or the vaguely disreputable. J. P. Morgan and\n",
      "Company, which (with Bonbright and Company) sponsored United Corporation in\n",
      "January 1929, offered a package of one share of common stock and one of\n",
      "preferred to a list of friends, Morgan partners included, at $75. This was a\n",
      "bargain. When trading in United Corporation began a week later, the price was 92\n",
      "bid, 94 asked on the over-the-counter market, and after four days the stock\n",
      "reached 99. Stock that had been taken up at 75 could be and was promptly resold\n",
      "at these prices.8 That such agreeable incentives greatly stimulated the\n",
      "organization of new investment trusts is hardly surprising.         IVThere were\n",
      "some, indeed, who only regretted that everyone could not participate in the\n",
      "gains from these new engines of financial progress. One of those who had\n",
      "benefited from the United Corporation promotion just mentioned was John J.\n",
      "Raskob. As Chairman of the Democratic National Committee, he was also\n",
      "politically committed to a firm friendship for the people. He believed that\n",
      "everyone should be in on the kind of opportunities he himself enjoyed.One of the\n",
      "fruits of this generous impulse during the year was an article in the Ladies'\n",
      "Home Journal with the attractive title, \"Everybody Ought to be Rich.\" In it Mr.\n",
      "Raskob pointed out that anyone who saved fifteen dollars a month, invested it in\n",
      "sound common stocks, and spent no dividends would be worth—as it then\n",
      "appeared—some eighty thousand dollars after twenty years. Obviously, at this\n",
      "rate, a great many people could be rich.But there was the twenty-year delay.\n",
      "Twenty years seemed a long time to get rich, especially in 1929, and for a\n",
      "Democrat and friend of the people to commit himself to such gradualism was to\n",
      "risk being thought a reactionary. Mr. Raskob, therefore, had a further\n",
      "suggestion. He proposed an investment trust which would be specifically designed\n",
      "to allow the poor man to increase his capital just as the rich man was doing.The\n",
      "plan, which Mr. Raskob released to the public in the early summer of 1929, was\n",
      "worked out in some detail. (The author stated that he had discussed it with\n",
      "\"financiers, economists, theorists, professors, bankers, labor leaders,\n",
      "industrial leaders, and many men of no prominence who have ideas.\") A company\n",
      "would be organized to buy stocks. The proletarian with, say, $200 would turn\n",
      "over his pittance to the company which would then buy stocks in the rather less\n",
      "meager amount of $500. The additional $300 the company would get from a\n",
      "financial subsidiary organized for the purpose, and with which it would post all\n",
      "of the stock as collateral. The incipient capitalist would pay off his debt at\n",
      "the rate of perhaps $25 a month. He would, of course, get the full benefit of\n",
      "the increase in the value of the stock, and this was something that Mr. Raskob\n",
      "regarded as inevitable. Hammering home the inadequacy of existing arrangements,\n",
      "Mr. Raskob said: \"Now all the man with $200 to $500 to invest can do today is to\n",
      "buy Liberty bonds...\"9The reaction to the Raskob plan was comparable to the\n",
      "response to a new and daring formulation of the relation of mass to energy. \"A\n",
      "practical Utopia,\" one paper called it. Another described it as \"The greatest\n",
      "vision of Wall Street's greatest mind.\" A tired and cynical commentator was\n",
      "moved to say that it looks \"more like financial statesmanship than anything that\n",
      "has come out of Wall Street in many a weary moon.\"10Had there been a little more\n",
      "time, it seems certain that something would have been made of Mr. Raskob's plan.\n",
      "People were full of enthusiasm for the wisdom and perspicacity of such men. This\n",
      "was admirably indicated by the willingness of people to pay for the genius of\n",
      "the professional financier.         VThe measure of this respect for financial\n",
      "genius was the relation of the market value of the outstanding securities of the\n",
      "investment trusts to the value of the securities it owned. Normally the\n",
      "securities of the trust were worth considerably more than the property it owned.\n",
      "Sometimes they were worth twice as much. There should be no ambiguity on this\n",
      "point. The only property of the investment trust was the common and preferred\n",
      "stocks and debentures, mortgages, bonds, and cash that it owned. (Often it had\n",
      "neither office nor office furniture; the sponsoring firm ran the investment\n",
      "trust out of its own quarters.) Yet, had these securities all been sold on the\n",
      "market, the proceeds would invariably have been less, and often much less, than\n",
      "the current value of the outstanding securities of the investment company. The\n",
      "latter, obviously, had some claim to value which went well beyond the assets\n",
      "behind them. That premium was, in effect, the value an admiring community placed\n",
      "on professional financial knowledge, skill, and manipulative ability. To value a\n",
      "portfolio of stocks \"at the market\" was to regard it only as inert property. But\n",
      "as the property of an investment trust it was much more, for the portfolio was\n",
      "then combined with the precious ingredient of financial genius. Such special\n",
      "ability could invoke a whole strategy for increasing the value of securities. It\n",
      "could join in pools and syndicates to put up values. It knew when others were\n",
      "doing likewise and could go along. Above all, the financial genius was in on\n",
      "things. It had access to what Mr. Lawrence of Princeton described as \"the stage\n",
      "whereon is focused the world's most intelligent and best informed judgment of\n",
      "the values of the enterprises which serve men's needs.\"11 One might make money\n",
      "investing directly in Radio, J. I. Case, or Montgomery Ward, but how much safer\n",
      "and wiser to let it be accomplished by the men of peculiar knowledge, and\n",
      "wisdom.By 1929 the investment trusts were aware of their reputation for\n",
      "omniscience, as well as its importance, and they lost no opportunity to enlarge\n",
      "it. To have a private economist was one possibility, and as the months passed a\n",
      "considerable competition developed for those men of adequate reputation and\n",
      "susceptibility. It was a golden age for professors. The American Founders Group,\n",
      "an awe-inspiring family of investment trusts, had as a director Professor Edwin\n",
      "W. Kemmerer, the famous Princeton money expert. The staff economist was Dr.\n",
      "Rufus Tucker, also a well-known figure. (That economists were not yet\n",
      "functioning with perfect foresight is perhaps suggested by the subsequent\n",
      "history of the enterprise. United Founders, the largest company in the group,\n",
      "suffered a net contraction in its assets of $301,385,504 by the end of 1935, and\n",
      "its stock dropped from a high of over $75 share in 1929 to a little under 75\n",
      "cents.)12 Still another great combine was advised by Dr. David Friday, who had\n",
      "come to Wall Street from the University of Michigan. Friday's reputation for\n",
      "both insight and foresight was breathtaking. A Michigan trust had three college\n",
      "professors—Irving Fisher of Yale, Joseph S. Davis of Stanford, and Edmund E. Day\n",
      "then of Michigan—to advise on its policies.13 The company stressed not only the\n",
      "diversity of its portfolio but also of its counsel. It was fully protected from\n",
      "any parochial Yale, Stanford, or Michigan view of the market.Other trusts urged\n",
      "the excellence of their genius in other terms. Thus one observed that, since it\n",
      "owned stocks in 120 corporations, it benefited from the \"combined efficiency of\n",
      "their presidents, officers, and the boards of directors.\" It noted further that\n",
      "\"closely allied to these corporations are the great banking institutions.\" Then,\n",
      "in something of a logical broad jump, it concluded, \"The trust, therefore,\n",
      "mobilizes to a large extent the successful business intellect of the country.\"\n",
      "Another concern, less skilled in logical method, contented itself with pointing\n",
      "out that \"Investing is a science instead of a 'one-man job.'\"14As 1929 wore\n",
      "along, it was plain that more and more of the new investors in the market were\n",
      "relying on the intellect and the science of the trusts. This meant, of course,\n",
      "that they still had the formidable problem of deciding between the good and the\n",
      "bad trusts. That there were some bad ones was (though barely) recognized.\n",
      "Writing in the March 1929 issue of The Atlantic Monthly, Paul C. Cabot stated\n",
      "that dishonesty, inattention, inability, and greed were among the common\n",
      "shortcomings of the new industry. These were impressive disadvantages, and as an\n",
      "organizer and officer of a promising investment trust, the State Street\n",
      "Investment Corporation, Mr. Cabot presumably spoke with some authority.15\n",
      "However, audience response to such warnings in 1929 was very poor. And the\n",
      "warnings were very infrequent.         VIKnowledge, manipulative skill, or\n",
      "financial genius were not the only magic of the investment trust. There was also\n",
      "leverage. By the summer of 1929, one no longer spoke of investment trusts as\n",
      "such. One referred to high-leverage trusts, low-leverage trusts, or trusts\n",
      "without any leverage at all.The principle of leverage is the same for an\n",
      "investment trust as in the game of crack-the-whip. By the application of well-\n",
      "known physical laws, a modest movement near the point of origin is translated\n",
      "into a major jolt on the extreme periphery. In an investment trust leverage was\n",
      "achieved by issuing bonds, preferred stock, as well as common stock to purchase,\n",
      "more or less exclusively, a portfolio of common stocks. When the common stock so\n",
      "purchased rose in value, a tendency which was always assumed, the value of the\n",
      "bonds and preferred stock of the trust was largely unaffected.16 These\n",
      "securities had a fixed value derived from a specified return. Most or all of the\n",
      "gain from rising portfolio values was concentrated on the common stock of the\n",
      "investment trust which, as a result, rose marvelously. Consider, by way of\n",
      "illustration, the case of an investment trust organized in early 1929 with a\n",
      "capital of $150 million—a plausible size by then. Let it be assumed, further,\n",
      "that a third of the capital was realized from the sale of bonds, a third from\n",
      "preferred stock, and the rest from the sale of common stock. If this $150\n",
      "million were invested, and if the securities so purchased showed a normal\n",
      "appreciation, the portfolio value would have increased by midsummer by about 50\n",
      "per cent. The assets would be worth $225 million. The bonds and preferred stock\n",
      "would still be worth only $100 million; their earnings would not have increased,\n",
      "and they could claim no greater share of the assets in the hypothetical event of\n",
      "a liquidation of the company. The remaining $125 million, therefore, would\n",
      "underlie the value of the common stock of the trust. The latter, in other words,\n",
      "would have increased in asset value from $50 million to $125 million, or by 150\n",
      "per cent, and as the result of an increase of only 50 per cent in the value of\n",
      "the assets of the trust as a whole.This was the magic of leverage, but this was\n",
      "not all of it. Were the common stock of the trust, which had so miraculously\n",
      "increased in value, held by still another trust with similar leverage, the\n",
      "common stock of that trust would get an increase of between 700 and 800 per cent\n",
      "from the original 50 per cent advance. And so forth. In 1929 the discovery of\n",
      "the wonders of the geometric series struck Wall Street with a force comparable\n",
      "to the invention of the wheel. There was a rush to sponsor investment trusts\n",
      "which would sponsor investment trusts, which would, in turn, sponsor investment\n",
      "trusts. The miracle of leverage, moreover, made this a relatively costless\n",
      "operation to the ultimate man behind all of the trusts. Having launched one\n",
      "trust and retained a share of the common stock, the capital gains from leverage\n",
      "made it relatively easy to swing a second and larger one which enhanced the\n",
      "gains and made possible a third and still bigger trust. Thus, Harrison Williams,\n",
      "one of the more ardent exponents of leverage, was thought by the Securities and\n",
      "Exchange Commission to have substantial influence over a combined investment\n",
      "trust and holding company system with a market value in 1929 at close to a\n",
      "billion dollars.17 This had been built on his original control of a smallish\n",
      "concern—the Central States Electric Corporation—which was worth only some six\n",
      "million dollars in 1921.18 Leverage was also a prime factor in the remarkable\n",
      "growth of the American Founders Group. The original member of this notable\n",
      "family of investment trusts was launched in 1921. The original promoter was,\n",
      "unhappily, unable to get the enterprise off the ground because he was in\n",
      "bankruptcy. However, the following year a friend contributed $500, with which\n",
      "modest capital a second trust was launched, and the two companies began\n",
      "business. The public reception was highly favorable, and by 1927 the two\n",
      "original companies and a third which had subsequently been added had sold\n",
      "between seventy and eighty million dollars worth of securities to the public.19\n",
      "But this was only the beginning; in 1928 and 1929 an explosion of activity\n",
      "struck the Founders Group. Stock was sold to the public at a furious rate. New\n",
      "firms with new names were organized to sell still more stock until, by the end\n",
      "of 1929, there were thirteen companies in the group. At that time the largest\n",
      "company, the United Founders Corporation, had total resources of $686,165,000.\n",
      "The group as a whole had resources with a market value of more than a billion\n",
      "dollars, which may well have been the largest volume of assets ever controlled\n",
      "by an original outlay of $500. Of the billion dollars, some $320,000,000 was\n",
      "represented by inter-company holdings—the investment of one or another company\n",
      "of the group in the securities of yet others. This fiscal incest was the\n",
      "instrument through which control was maintained and leverage enjoyed. Thanks to\n",
      "this long chain of holdings by one company in another, the increases in values\n",
      "in 1928 and 1929 were effectively concentrated in the value of the common stock\n",
      "of the original companies.Leverage, it was later to develop, works both ways.\n",
      "Not all of the securities held by the Founders were of a kind calculated to rise\n",
      "indefinitely, much less to resist depression. Some years later the portfolio was\n",
      "found to have contained 5000 shares of Kreuger and Toll, 20,000 shares of Kolo\n",
      "Products Corporation, an adventuresome new company which was to make soap out of\n",
      "banana oil, and $295,000 in the bonds of the Kingdom of Yugoslavia.20 As Kreuger\n",
      "and Toll moved down to its ultimate value of nothing, leverage was also at\n",
      "work—geometric series are equally dramatic in reverse. But this aspect of the\n",
      "mathematics of leverage was still unrevealed in early 1929, and notice must\n",
      "first be taken of the most dramatic of all the investment company promotions of\n",
      "that remarkable year, those of Goldman, Sachs.         VIIGoldman, Sachs and\n",
      "Company, an investment banking and brokerage partnership, came rather late to\n",
      "the investment trust business. Not until December 4, 1928, less than a year\n",
      "before the stock market crash, did it sponsor the Goldman Sachs Trading\n",
      "Corporation, its initial venture in the field. However, rarely, if ever, in\n",
      "history has an enterprise grown as the Goldman Sachs Trading Corporation and its\n",
      "offspring grew in the months ahead.The initial issue of stock in the Trading\n",
      "Corporation was a million shares, all of which was bought by Goldman, Sachs and\n",
      "Company at $100 a share for a total of $100,000,000. Ninety per cent was then\n",
      "sold to the public at $104. There were no bonds and no preferred stocks;\n",
      "leverage had not yet been discovered by Goldman, Sachs and Company. Control of\n",
      "the Goldman Sachs Trading Corporation remained with Goldman, Sachs and Company\n",
      "by virtue of a management contract and the presence of the partners of the\n",
      "company on the board of the Trading Corporation.21In the two months after its\n",
      "formation, the new company sold some more stock to the public, and on February\n",
      "21 it merged with another investment trust, the Financial and Industrial\n",
      "Securities Corporation. The assets of the resulting company were valued at $235\n",
      "million, reflecting a gain of well over 100 per cent in under three months. By\n",
      "February 2, roughly three weeks before the merger, the stock for which the\n",
      "original investors had paid $104 was selling for $136.50. Five days later, on\n",
      "February 7, it reached $222.50. At this latter figure it had a value\n",
      "approximately twice that of the current total worth of the securities, cash, and\n",
      "other assets owned by the Trading Corporation. This remarkable premium was not\n",
      "the undiluted result of public enthusiasm for the financial genius of Goldman,\n",
      "Sachs. Goldman, Sachs had considerable enthusiasm for itself, and the Trading\n",
      "Corporation was buying heavily of its own securities. By March 14 it had bought\n",
      "560,724 shares of its own stock for a total outlay of $57,021,936.22 This, in\n",
      "turn, had boomed their value. However, perhaps foreseeing the exiguous character\n",
      "of an investment company which had its investments all in its own common stock,\n",
      "the Trading Corporation stopped buying itself in March. Then it resold part of\n",
      "the stock to William Crapo Durant, who re-resold it to the public as opportunity\n",
      "allowed.The spring and early summer were relatively quiet for Goldman, Sachs,\n",
      "but it was a period of preparation. By July 26 it was ready. On that date the\n",
      "Trading Corporation, jointly with Harrison Williams, launched the Shenandoah\n",
      "Corporation, the first of two remarkable trusts. The initial securities issue by\n",
      "Shenandoah was $102,500,000 (there was an additional issue a couple of months\n",
      "later) and it was reported to have been oversubscribed some sevenfold. There\n",
      "were both preferred and common stock, for by now Goldman, Sachs knew the\n",
      "advantages of leverage. Of the five million shares of common stock in the\n",
      "initial offering, two million were taken by the Trading Corporation, and two\n",
      "million by Central States Electric Corporation on behalf of the cosponsor,\n",
      "Harrison Williams. Williams was a member of the small board along with partners\n",
      "in Goldman, Sachs. Another board member was a prominent New York attorney whose\n",
      "lack of discrimination in this instance may perhaps be attributed to youthful\n",
      "optimism. It was Mr. John Foster Dulles. The stock of Shenandoah was issued at\n",
      "$17.50. There was brisk trading on a \"when issued\" basis. It opened at 30,\n",
      "reached a high of 36, and closed at 36, or 18.5 above the issue price. (By the\n",
      "end of the year the price was 8 and a fraction. It later touched fifty cents.)\n",
      "Meanwhile Goldman, Sachs was already preparing its second tribute to the\n",
      "countryside of Thomas Jefferson, the prophet of small and simple enterprises.\n",
      "This was the even mightier Blue Ridge Corporation, which made its appearance on\n",
      "August 20. Blue Ridge had a capital of $142,000,000, and nothing about it was\n",
      "more remarkable than the fact that it was sponsored by Shenandoah, its precursor\n",
      "by precisely twenty-five days. Blue Ridge had the same board of directors as\n",
      "Shenandoah, including the still optimistic Mr. Dulles, and of its 7,250,000\n",
      "shares of common stock (there was also a substantial issue of preferred)\n",
      "Shenandoah subscribed a total of 6,250,000. Goldman, Sachs by now was applying\n",
      "leverage with a vengeance.An interesting feature of Blue Ridge was the\n",
      "opportunity it offered the investor to divest himself of routine securities in\n",
      "direct exchange for the preferred and common stock of the new corporation. A\n",
      "holder of American Telephone and Telegraph Company could receive 4 Wis shares\n",
      "each of Blue Ridge Preference and Common for each share of Telephone stock\n",
      "turned in. The same privilege was extended to holders of Allied Chemical and\n",
      "Dye, Santa Fe, Eastman Kodak, General Electric, Standard Oil of New Jersey, and\n",
      "some fifteen other stocks. There was much interest in this offer. August 20, the\n",
      "birthday of Blue Ridge, was a Tuesday, but there was more work to be done by\n",
      "Goldman, Sachs that week. On Thursday, the Goldman Sachs Trading Corporation\n",
      "announced the acquisition of the Pacific American Associates, a West Coast\n",
      "investment trust which, in turn, had recently bought a number of smaller\n",
      "investment trusts and which owned the American Trust Company, a large commercial\n",
      "bank with numerous branches throughout California. Pacific American had a\n",
      "capital of around a hundred million. In preparation for the merger, the Trading\n",
      "Corporation had issued another $71,400,000 in stock which it had exchanged for\n",
      "capital stock of the American Company, the holding company which owned over 99\n",
      "per cent of the common stock of the American Trust Company.23Having issued more\n",
      "than a quarter of a billion dollars worth of securities in less than a month—an\n",
      "operation that would not then have been unimpressive for the United States\n",
      "Treasury—activity at Goldman, Sachs subsided somewhat. Its members had not been\n",
      "the only busy people during this time. It was a poor day in August and September\n",
      "of that year when no new trust was announced or no large new issue of securities\n",
      "was offered by an old one. Thus, on August first, the papers announced the\n",
      "formation of Anglo-American Shares, Inc., a company which, with a soigné touch\n",
      "not often seen in a Delaware corporation, had among its directors the Marquess\n",
      "of Carisbrooke, GCB, GCVO, and Colonel, the Master of Sempill, AFC, otherwise\n",
      "identified as the President of the Royal Aeronautical Society, London. American\n",
      "Insuranstocks Corporation was launched the same day, though boasting no more\n",
      "glamorous a director than William Cibbs McAdoo. On succeeding days came Gude\n",
      "Winmill Trading Corporation, National Republic Investment Trust, Insull Utility\n",
      "Investments, Inc., International Carriers, Ltd., Tri-Continental Allied\n",
      "Corporation, and Solvay American Investment Corporation. On August 13 the papers\n",
      "also announced that an Assistant U.S. Attorney had visited the offices of the\n",
      "Cosmopolitan Fiscal Corporation and also an investment service called the\n",
      "Financial Counselor. In both cases the principals were absent. The offices of\n",
      "the Financial Counselor were equipped with a peephole like a speakeasy. More\n",
      "investment trust securities were offered in September of 1929 even than in\n",
      "August—the total was above $600 million.24 However, the nearly simultaneous\n",
      "promotion of Shenandoah and Blue Ridge was to stand as the pinnacle of new era\n",
      "finance. It is difficult not to marvel at the imagination which was implicit in\n",
      "this gargantuan insanity. If there must be madness something may be said for\n",
      "having it on a heroic scale.Years later, on a gray dawn in Washington, the\n",
      "following colloquy occurred before a committee of the United States\n",
      "Senate.25 Senator Couzens. Did Goldman, Sachs and Company organize the Goldman\n",
      "Sachs Trading Corporation?Mr. Sachs. Yes, sir. Senator Couzens. And it sold its\n",
      "stock to the public?Mr. Sachs. A portion of it. The firm invested originally in\n",
      "10 per cent of the entire issue for the sum of $10,000,000.Senator Couzens. And\n",
      "the other 90 per cent was sold to the public?Mr. Sachs. Yes, sir.Senator\n",
      "Couzens. At what price?Mr. Sachs. At 104. That is the old stock ... the stock\n",
      "was split two for one.Senator Couzens. And what is the price of the stock\n",
      "now?Mr. Sachs. Approximately 1⅓.         CHAPTER IVThe Twilight of IllusionTHERE\n",
      "WAS no summer lull in Wall Street that year. Along with the great investment\n",
      "trust promotions went the greatest market ever. Every day prices rose; they\n",
      "almost never fell. During June the Times industrials went up 52 points; in July\n",
      "they gained another 25. This was a total gain of 77 points in two months. In all\n",
      "of the remarkable year of 1928 they had gone up only 86.5 points. Then in August\n",
      "they rose another 33 points. This gain of 110 points in three months—from 339 on\n",
      "the last day of May to 449 on the last day of August—meant that during the\n",
      "summer values had increased, over-all, by nearly a quarter.Individual issues had\n",
      "also done very well. During the three summer months, Westinghouse went from 151\n",
      "to 286 for a net gain of 135. General Electric was up from 268 to 391, and Steel\n",
      "from 165 to 258. Even so somber a security as American Tel and Tel had gone from\n",
      "209 to 303. The investment trusts were making good gains. United Founders went\n",
      "from 36 bid to 68; Alleghany Corporation rose from 33 to 56.The volume of\n",
      "trading was also consistently heavy. On the New York Stock Exchange it was\n",
      "frequently between four and five million shares. Only occasionally on a full day\n",
      "did it drop below three million. However, trading on the New York Stock Exchange\n",
      "was no longer a good index of the total interest in securities speculation. Many\n",
      "new and exciting issues—Shenandoah, Blue Ridge, Pennroad, Insull Utilities—were\n",
      "not listed on the Big Board. The New York Stock Exchange in those days was not a\n",
      "snobbish, prying, or an intolerant institution. Most companies that so desired\n",
      "could have their stocks listed. Nevertheless, there were some who found it wise,\n",
      "and many more who found it convenient not to answer the fairly elementary\n",
      "requests which were made by the Exchange for information. The new stocks,\n",
      "accordingly, were traded on the Curb, or in Boston, or on other out-of-town\n",
      "exchanges. Although business on the New York Stock Exchange remained larger than\n",
      "that on all other markets combined, its relative position suffered. (In 1929 it\n",
      "had an estimated 61 per cent of all transactions; three years later, when most\n",
      "of the new trusts had disappeared forever, the Exchange had 76 per cent of the\n",
      "total business.)1 It follows that by the summer of 1929 the normally\n",
      "somnambulant markets of Boston, San Francisco, and even of Cincinnati were\n",
      "having a boom. Instead of being merely a pale reflection of the real thing on\n",
      "Wall Street, they had a life and personality of their own. Stocks were for sale\n",
      "here that couldn't be had in New York, and some of them had an exceptional\n",
      "speculative lack. By 1929 it was a poor town, sadly devoid of civic spirit,\n",
      "which wasn't wondering if it too shouldn't have a stock market. More than the\n",
      "prices of common stocks were rising. So, at an appalling rate, was the volume of\n",
      "speculation. Brokers' loans during the summer increased at a rate of about\n",
      "$400,000,000 a month. By the end of the summer, the total exceeded seven\n",
      "billions. Of that more than half was being supplied by corporations and\n",
      "individuals, at home and abroad, who were taking advantage of the excellent rate\n",
      "of return which New York was providing on money. Only rarely did the rate on\n",
      "call loans during that summer get as low as six per cent. The normal range was\n",
      "seven to twelve. On one occasion the rate touched fifteen. Since, as earlier\n",
      "observed, these loans provided all but total safety, liquidity, and ease of\n",
      "administration, the interest would not have seemed unattractive to a usurious\n",
      "moneylender in Bombay. To a few alarmed observers it seemed as though Wall\n",
      "Street were by way of devouring all the money of the entire world. However, in\n",
      "accordance with the cultural practice, as the summer passed, the sound and\n",
      "responsible spokesmen decried not the increase in brokers' loans, but those who\n",
      "insisted on attaching significance to this trend. There was a sharp criticism of\n",
      "the prophets of doom.         IIThere were two sources of intelligence on\n",
      "brokers' loans. One was the monthly tabulation of the New York Stock Exchange,\n",
      "which in general is used here. The other was the slightly less complete return\n",
      "of the Federal Reserve System which was published weekly. Each Friday this\n",
      "report showed a large increase in loans; each Friday it was firmly stated that\n",
      "it didn't mean a thing, and anyone who suggested otherwise was administered a\n",
      "stern rebuke. It seems probable that only a minority of the people in the market\n",
      "related the volume of the brokers' loans to the volume of purchases on margin\n",
      "and thence to the amount of speculation. Accordingly, an expression of concern\n",
      "over these loans was easily attacked as a gratuitous effort to undermine\n",
      "confidence. Thus, in Barron's on July 8, Sheldon Sinclair Wells explained that\n",
      "those who worried about brokers' loans, and about the influx of funds from\n",
      "corporations, simply did not know what was going on. The call market had become\n",
      "a great new investment outlet for corporate reserves, he argued. The critics did\n",
      "not appreciate this change. Chairman Mitchell of the National City Bank, by\n",
      "nature an equable man, was repeatedly angered by the attention being given to\n",
      "brokers' loans and expressed himself strongly. The financial press was also\n",
      "disturbed, and when Arthur Brisbane later in the year questioned the propriety\n",
      "of a 10 per cent call rate, The Wall Street Journal reached the end of its\n",
      "patience. \"Even in general newspapers some accurate knowledge is required for\n",
      "discussing most things. Why is it that any ignoramus can talk about Wall\n",
      "Street?\"2 (It does seem possible that Brisbane thought the rate was 10 per cent\n",
      "a day rather than a year.)Scholars also reacted against those who, deliberately\n",
      "or otherwise, were sabotaging prosperity with their unguarded pessimism. After\n",
      "soberly viewing the situation, Professor Dice concluded that the high level of\n",
      "brokers' loans should not be \"as greatly feared as some would have us believe.\"\n",
      "3 In August the Midland Bank of Cleveland made public the results of\n",
      "calculations which proved that until loans by corporations in the stock market\n",
      "reached twelve billion there was no cause for concern.4The best reassurance on\n",
      "brokers' loans was in the outlook for the market. If stocks remained high and\n",
      "went higher, and if they did so because their prospects justified their price,\n",
      "then there was no occasion to worry about the loans that were piling up.\n",
      "Accordingly, much of the defense of the loans consisted in defending the levels\n",
      "of the market. It was not hard to persuade people that the market was sound; as\n",
      "always in such times they asked only that the disturbing voices of doubt be\n",
      "muted and that there be tolerably frequent expressions of confidence. In 1929\n",
      "treason had not yet become a casual term of reproach. As a result, pessimism was\n",
      "not openly equated with efforts to destroy the American way of life. Yet it had\n",
      "such connotations. Almost without exception, those who expressed concern said\n",
      "subsequently that they did so with fear and trepidation. (Later in the year a\n",
      "Boston firm of investment counselors struck a modern note with a widely\n",
      "publicized warning that America had no place for \"destructionists.\")The official\n",
      "optimists were many and articulate. Thus in June, Bernard Baruch told Bruce\n",
      "Barton, in a famous interview published in The American Magazine5 that \"the\n",
      "economic condition of the world seems on the verge of a great forward movement.\"\n",
      "He pointed out that no bears had houses on Fifth Avenue. Numerous college\n",
      "professors also exuded scientific confidence. In light of later developments,\n",
      "the record of the Ivy League was especially unfortunate. In a statement which\n",
      "achieved minor notoriety, Lawrence of Princeton said that \"the consensus of\n",
      "judgment of the millions whose valuations function on that admirable market, the\n",
      "Stock Exchange, is that stocks are not at present over-valued.\" He added: \"Where\n",
      "is that group of men with the all-embracing wisdom which will entitle them to\n",
      "veto the judgment of this intelligent multitude?\"6That autumn Professor Irving\n",
      "Fisher of Yale made his immortal estimate: \"Stock prices have reached what looks\n",
      "like a permanently high plateau.\" Irving Fisher was the most original of\n",
      "American economists. Happily there are better things —his contributions to index\n",
      "numbers, technical economic theory, and monetary theory—for which he is\n",
      "remembered.From Cambridge slightly less spacious reassurance came from the\n",
      "Harvard Economic Society, an extracurricular enterprise conducted by a number of\n",
      "economics professors of unexceptionable conservatism. The purpose of the Society\n",
      "was to help businessmen and speculators foretell the future. Forecasts were made\n",
      "several times a month and undoubtedly gained in stature from their association\n",
      "with the august name of the university.By wisdom or good luck, the Society in\n",
      "early 1929 was mildly bearish. Its forecasters had happened to decide that a\n",
      "recession (though assuredly not a depression) was overdue. Week by week they\n",
      "foretold a slight setback in business. When, by the summer of 1929, the setback\n",
      "had not appeared, at least in any very visible form, the Society gave up and\n",
      "confessed error. Business, it decided, might be good after all. This, as such\n",
      "things are judged, was still a creditable record, but then came the crash. The\n",
      "Society remained persuaded that no serious depression was in prospect. In\n",
      "November it said firmly that \"a severe depression like that of 1920–21 is\n",
      "outside the range of probability. We are not facing protracted liquidation.\"\n",
      "This view the Society reiterated until it was liquidated.         IIIThe bankers\n",
      "were also a source of encouragement to those who wished to believe in the\n",
      "permanence of the boom. A great many of them abandoned their historic role as\n",
      "the guardians of the nation's fiscal pessimism and enjoyed a brief respite of\n",
      "optimism. They had reasons for doing so. In the years preceding, a considerable\n",
      "number of the commercial banks, including the largest of the New York houses,\n",
      "had organized securities affiliates. These affiliates sold stocks and bonds to\n",
      "the public, and this business had become important. It was a business that\n",
      "compelled a rosy view of the future. In addition, individual bankers, perhaps\n",
      "taking a cue from the heads of the National City and Chase in New York, were\n",
      "speculating vigorously on their own behalf. They were unlikely to say, much less\n",
      "to advocate, anything that would jar the market. However, there were exceptions.\n",
      "One was Paul M. Warburg of the International Acceptance Bank, whose predictions\n",
      "must be accorded the same prominence as the forecasts of Irving Fisher. They\n",
      "were remarkably prescient. In March of 1929, he called for a stronger Federal\n",
      "Reserve policy and argued that if the present orgy of \"unrestrained speculation\"\n",
      "were not brought promptly to a halt there would ultimately be a disastrous\n",
      "collapse. This, he suggested, would be unfortunate not alone for the\n",
      "speculators. It would \"bring about a general depression involving the entire\n",
      "country.\"7Only Wall Street spokesmen who took the most charitable view of\n",
      "Warburg contented themselves with describing him as obsolete. One said he was\n",
      "\"sandbagging American prosperity.\" Others hinted that he had a motive—presumably\n",
      "a short position. As the market went up and up, his warnings were recalled only\n",
      "with contempt.8The most notable skeptics were provided by the press. They were a\n",
      "great minority to be sure. Most magazines and most newspapers in 1929 reported\n",
      "the upward sweep of the market with admiration and awe and without alarm. They\n",
      "viewed both the present and the future with exuberance. Moreover, by 1929\n",
      "numerous journalists were sternly resisting the more subtle blandishments and\n",
      "flattery to which they have been thought susceptible. Instead they were\n",
      "demanding cold cash for news favorable to the market. A financial columnist of\n",
      "the Daily News, who signed himself \"The Trader,\" received some $19,000 in 1929\n",
      "and early 1930 from a free-lance operator named John J. Levenson. \"The Trader\"\n",
      "repeatedly spoke well of stocks in which Mr. Levenson was interested. Mr.\n",
      "Levenson later insisted, however, that this was a coincidence and that the\n",
      "payment reflected his more or less habitual generosity.9 A radio commentator\n",
      "named William J. McMahon was the president of the McMahon Institute of Economic\n",
      "Research, an organization that was mostly McMahon. He told in his broadcasts of\n",
      "the brilliant prospects of stocks which pool operators were seeking to boom. For\n",
      "this, it later developed, he received an honorarium of $250 a week from a\n",
      "certain David M. Lion.10 Mr. Lion was one of several whom the Pecora Committee\n",
      "reported as making a business of buying favorable comment in the necessary\n",
      "amount at the proper moment. At the other extreme was the best of the financial\n",
      "press. The established financial services like Poor's and that of the Standard\n",
      "Statistics Company never lost touch with reality. In the autumn Poor's Weekly\n",
      "Business and Investment Letter went so far as to speak of the \"great common-\n",
      "stock delusion.\" 11 The editor of The Commercial and Financial Chronicle was\n",
      "never quite shaken in his conviction that Wall Street had taken leave of its\n",
      "senses. The weekly reports on the brokers' loans were regularly the occasion for\n",
      "a solemn warning; the news columns featured any available bad news. However, by\n",
      "far the greatest force for sobriety was the New York Times. Under the guidance\n",
      "of the veteran Alexander Dana Noyes, its financial page was all but immune to\n",
      "the blandishments of the New Era. A regular reader could not doubt that a day of\n",
      "reckoning was expected. Also, on several occasions it reported, much too\n",
      "prematurely, that the day of reckoning had arrived.Indeed the temporary breaks\n",
      "in the market which preceded the crash were a serious trial for those who had\n",
      "declined fantasy. Early in 1928, in June, in December, and in February and March\n",
      "of 1929 it seemed that the end had come. On various of these occasions the Times\n",
      "happily reported the return to reality. And then the market took flight again.\n",
      "Only a durable sense of doom could survive such discouragement. The time was\n",
      "coming when the optimists would reap a rich harvest of discredit. But it has\n",
      "long since been forgotten that for many months those who resisted reassurance\n",
      "were similarly, if less permanently, discredited. To say that the Times, when\n",
      "the real crash came, reported the event with jubilation would be an\n",
      "exaggeration. Nevertheless, it covered it with an unmistakable absence of\n",
      "sorrow.         IVBy the summer of 1929 the market not only dominated the news.\n",
      "It also dominated the culture. That recherché minority which at other times has\n",
      "acknowledged its interest in Saint Thomas Aquinas, Proust, psychoanalysis, and\n",
      "psychosomatic medicine then spoke of United Corporation, United Founders, and\n",
      "Steel. Only the most aggressive of the eccentrics maintained their detachment\n",
      "from the market and their interest in autosuggestion or communism. Main Street\n",
      "had always had one citizen who could speak knowingly about buying or selling\n",
      "stocks. Now he became an oracle. In New York, on the edge of any gathering of\n",
      "significantly interesting people there had long been a literate broker or\n",
      "investment counselor who was abreast of current plans for pools, syndicates, and\n",
      "mergers, and was aware of attractive possibilities. He helpfully advised his\n",
      "friends on investments, and pressed, he would always tell what he knew of the\n",
      "market and much that he didn't. Now these men, even in the company of artists,\n",
      "playwrights, poets, and beautiful concubines, suddenly shone forth. Their words,\n",
      "more or less literally, became golden. Their audience listened not with the\n",
      "casual heed of people who are collecting quotable epigrams, but with the truly\n",
      "rapt attention of those who expect to make money by what they hear. That much of\n",
      "what was repeated about the market—then as now—bore no relation to reality is\n",
      "important, but not remarkable. Between human beings there is a type of\n",
      "intercourse which proceeds not from knowledge, or even from lack of knowledge,\n",
      "but from failure to know what isn't known. This was true of much of the\n",
      "discourse on the market. At luncheon in downtown Scranton, the knowledgeable\n",
      "physician spoke of the impending split-up in the stock of Western Utility\n",
      "Investors and the effect on prices. Neither the doctor nor his listeners knew\n",
      "why there should be a split-up, why it should increase values, or even why\n",
      "Western Utility Investors should have any value. But neither the doctor nor his\n",
      "audience knew that he did not know. Wisdom, itself, is often an abstraction\n",
      "associated not with fact or reality but with the man who asserts it and the\n",
      "manner of its assertion.Perhaps the failure to visualize the extent of one's\n",
      "innocence was especially true of women investors, who by now were entering the\n",
      "market in increasing numbers. (An article in The North American Review in April\n",
      "reported that women had become important players of \"man's most exciting\n",
      "capitalistic game\" and that the modern housewife now \"reads, for instance, that\n",
      "Wright Aero is going up ... just as she does that fresh fish is now on the\n",
      "market...\" The author hazarded the guess that success in speculation would do a\n",
      "lot for women's prestige.) To the typical female plunger the association of\n",
      "Steel was not with a corporation, and certainly not with mines, ships,\n",
      "railroads, blast furnaces, and open hearths. Rather it was with symbols on a\n",
      "tape and lines on a chart and a price that went up. She spoke of Steel with the\n",
      "familiarity of an old friend, when in fact she knew nothing of it whatever. Nor\n",
      "would anyone tell her that she did not know that she did not know. We are a\n",
      "polite and cautious people, and we avoid unpleasantness. Moreover, such advice,\n",
      "so far from accomplishing any result, would only have inspired a feeling of\n",
      "contempt for anyone who lacked the courage and the initiative and the\n",
      "sophistication to see how easily one could become rich. The lady operator had\n",
      "discovered she could be rich. Surely her right to be rich was as good as\n",
      "anyone's. One of the uses of women is that their motivations, though often\n",
      "similar, are less elaborately disguised than those of men.The values of a\n",
      "society totally preoccupied with making money are not altogether reassuring.\n",
      "During the summer, the Times accepted the copy of a dealer in the securities of\n",
      "the National Waterworks Corporation, a company which had been organized to buy\n",
      "into city water companies. The advertisement carried the following acquisitive\n",
      "thought: \"Picture this scene today, if by some cataclysm only one small well\n",
      "should remain for the great city of New York—$1.00 a bucket, $100, $1,000,\n",
      "$1,000,000. The man who owned the well would own the wealth of the city.\" All\n",
      "cataclysmically minded investors were urged to get a long position in water\n",
      "before it was too late.         VThe polar role of the stock market in American\n",
      "life in the summer of 1929 is beyond doubt. And many people of many different\n",
      "kinds and condition were in the stock market. Frederick Lewis Allen pictured the\n",
      "diversity of this participation in a fine passage: The rich man's chauffeur\n",
      "drove with his ears laid back to catch the news of an impending move in\n",
      "Bethlehem Steel; he held fifty shares himself on a twenty-point margin. The\n",
      "window-cleaner at the broker's office paused to watch the ticker, for he was\n",
      "thinking of converting his laboriously accumulated savings into a few shares of\n",
      "Simmons. Edwin Lefèvre (an articulate reporter on the market at this time who\n",
      "could claim considerable personal experience ) told of a broker's valet who made\n",
      "nearly a quarter of a million in the market, of a trained nurse who cleaned up\n",
      "thirty thousand following the tips given her by grateful patients; and of a\n",
      "Wyoming cattleman, thirty miles from the nearest railroad, who bought or sold a\n",
      "thousand shares a day.12 Yet there is probably more danger of overestimating\n",
      "rather than underestimating the popular interest in the market. The cliché that\n",
      "by 1929 everyone \"was in the market\" is far from the literal truth. Then, as\n",
      "now, to the great majority of workers, farmers, white-collar workers, indeed to\n",
      "the great majority of all Americans, the stock market was a remote and vaguely\n",
      "ominous thing. Then, as now, not many knew how one went about buying a security;\n",
      "the purchase of stocks on margin was in every respect as remote from life as the\n",
      "casino at Monte Carlo.In later years, a Senate committee investigating the\n",
      "securities markets undertook to ascertain the number of people who were involved\n",
      "in securities speculation in 1929. The member firms of twenty-nine exchanges in\n",
      "that year reported themselves as having accounts with a total of 1,548,–707\n",
      "customers. (Of these, 1,371,920 were customers of member firms of the New York\n",
      "Stock Exchange.) Thus only one and a half million people, out of a population of\n",
      "approximately 120 million and of between 29 and 30 million families, had an\n",
      "active association of any sort with the stock market. And not all of these were\n",
      "speculators. Brokerage firms estimated for the Senate committee that only about\n",
      "600,000 of the accounts just mentioned were for margin trading, as compared with\n",
      "roughly 950,000 in which trading was for cash. The figure of 600,000 for margin\n",
      "traders involves some duplication—a few large operators had accounts with more\n",
      "than one broker. There were also some traders whose operations were\n",
      "insignificant. However, some speculators are included among the 950,000 cash\n",
      "customers. Some were putting up the full purchase price for their securities,\n",
      "although speculating nonetheless. Some were borrowing money outside the market\n",
      "and posting the securities as collateral. Though listed as cash customers, they\n",
      "were in effect buying on margin. However, it is safe to say that at the peak in\n",
      "1929 the number of active speculators was less—and probably was much less—than a\n",
      "million. Between the end of 1928 and the end of July of 1929, a period when the\n",
      "popular folklore has Americans rushing like lemmings to participate in the\n",
      "market, the number of margin accounts on all of the exchanges of the country\n",
      "increased by only slightly more than fifty thousand.13 Hie striking thing about\n",
      "the stock market speculation of 1929 was not the massiveness of the\n",
      "participation. Rather it was the way it became central to the culture.\n",
      "VIBy the end of the summer of 1929, brokers' bulletins and letters no longer\n",
      "contented themselves with saying what stocks would rise that day and by how\n",
      "much. They went on to say that at 2 P.M. Radio or General Motors would be \"taken\n",
      "in hand.\"14 The conviction that the market had become the personal instrument of\n",
      "mysterious but omnipotent men was never stronger. And, indeed, this was a period\n",
      "of exceedingly active pool and syndicate operations—in short, of manipulation.\n",
      "During 1929 more than a hundred issues on the New York Stock Exchange were\n",
      "subject to manipulative operations, in which members of the Exchange or their\n",
      "partners had participated. The nature of these operations varied somewhat but,\n",
      "in a typical operation, a number of traders pooled their resources to boom a\n",
      "particular stock. They appointed a pool manager, promised not to double-cross\n",
      "each other by private operations, and the pool manager then took a position in\n",
      "the stock which might also include shares contributed by the participants. This\n",
      "buying would increase prices and attract the interest of people watching the\n",
      "tape across the country. The interest of the latter would then be further\n",
      "stimulated by active selling and buying, all of which gave the impression that\n",
      "something big was afloat. Tipsheets and market commentators would tell of\n",
      "exciting developments in the offing. If all went well, the public would come in\n",
      "to buy, and prices would rise on their own. The pool manager would then sell\n",
      "out, pay himself a percentage of the profits, and divide the rest with his\n",
      "investors.15While it lasted, there was never a more agreeable way of making\n",
      "money. The public at large sensed the attractiveness of these operations, and as\n",
      "the summer passed it came to be supposed that Wall Street was concerned with\n",
      "little else. This was an exaggeration, but it did not discourage public activity\n",
      "in the market. People did not believe they were being shorn. Nor were they. Both\n",
      "they and the pool operators were making money with the difference, only, that\n",
      "the latter were making more. In any case, the public reaction to inside\n",
      "operations was to hope that it might get some inside information on these\n",
      "operations and so get a cut in the profits that the great men like Cutten,\n",
      "Livermore, Raskob, and the rest were making. As the market came to be considered\n",
      "less and less a long-run register of corporate prospects and more and more a\n",
      "product of manipulative artifice, the speculator was required to give it his\n",
      "closest, and preferably his undivided attention. Signs of incipient pool\n",
      "activity had to be detected at the earliest possible moment, which meant that\n",
      "one needed to have his eyes on the tape. However, even the person who was\n",
      "relying on hunches, incantations, or simple faith, as distinct from the effort\n",
      "to assess the intentions of the professionals, found it hard to be out of touch.\n",
      "Only in the case of the rarest individuals can speculation be a part-time\n",
      "activity. Money for most people is far too important. Of the South Sea Bubble it\n",
      "was observed that \"Statesmen forgot their Politics, Lawyers the Bar, Merchants\n",
      "their Traffic, Physicians their Patients, Tradesmen their Shops, Debtors of\n",
      "Quality their Creditors, Divines the Pulpit, and even the Women themselves their\n",
      "Pride and Vanity!\" 16 And so it was in 1929. \"Brokers' offices were crowded from\n",
      "10 A.M. to 3 P.M. with seated or standing customers who, instead of attending to\n",
      "their own business, were watching the blackboard. In some 'customers rooms' it\n",
      "was difficult to get access to a spot from which the posted quotations could be\n",
      "seen; no one could get a chance to inspect the tape.\" 17 It follows that to be\n",
      "out of touch with the market, ever so briefly, was a nerve-wracking experience.\n",
      "Happily, this was not often necessary. Ticker service was now nationwide; a\n",
      "local telephone call would get the latest quotations from almost anywhere. A\n",
      "journey to Europe provided one of the few troublesome exceptions. As The\n",
      "Literary Digest pointed out during the course of the summer, \"Transoceanic\n",
      "brokerage business has been growing to immense proportions ... But there has\n",
      "been an interlude of uncertainty and inconvenience for speculators crossing the\n",
      "ocean.\" 18 In August even this interlude was eliminated. Progressive brokerage\n",
      "houses—a leader was M. J. Meehan, the specialist in Radio and a veteran of many\n",
      "notable manipulations—installed branches on the big ships under special\n",
      "regulations laid down by the Exchange. On August 17, the Leviathan and the lie\n",
      "de France left port fully equipped for speculation on the high seas. Business on\n",
      "the lie the opening day was described as brisk. One of the first transactions\n",
      "was by Irving Berlin, who sold 1000 shares of Paramount-Famous-Lasky at 72. (It\n",
      "was a shrewd move. The stock later went more or less to nothing and the company\n",
      "into bankruptcy.)In Spokane an anonymous poet on the editorial staff of the\n",
      "Spokesman-Review celebrated the seagoing boardrooms. We were crowded in the\n",
      "cabin Watching figures on the Board; It was midnight on the ocean And a tempest\n",
      "loudly roared. *** \"We are lost!\" the Captain shouted, As he staggered down the\n",
      "stairs. I've got a tip,\" he faltered, \"Straight by wireless from the aunt Of a\n",
      "fellow who's related To a cousin of Durant.\" At these awful words we shuddered,\n",
      "And the stoutest bull grew sick While the brokers cried, \"More margin!\" And the\n",
      "ticker ceased to tick. But the captain's little daughter Said, \"I do not\n",
      "understand— Isn't Morgan on the ocean Just the same as on the land?\"19\n",
      "VIILabor Day brought the summer of 1929 to its conventional end on September 2.\n",
      "There was a severe heat wave, and on the evening of the holiday returning\n",
      "motorists tied up the roads around New York for miles. In the end many were\n",
      "forced to abandon their vehicles and make their way home by train or subway. On\n",
      "September 3 the city continued to swelter in what the Weather Bureau reported\n",
      "was the hottest day of the year.Away from Wall Street this was a very quiet day\n",
      "in a very tranquil time. Years later Frederick Lewis Allen went to the\n",
      "newspapers for that day and in a charming article told all that he found.20\n",
      "There wasn't much. Disarmament was being discussed in that customarily desultory\n",
      "fashion which doubtless in the end will destroy us. The Graf Zeppelin was\n",
      "nearing the end of its first round-the-world flight. A tri-motored plane of\n",
      "Transcontinental Air Transport had crashed in a thunderstorm in New Mexico, and\n",
      "eight were killed. (The line had only recently inaugurated a forty-eight-hour\n",
      "service to the West Coast—railway sleeper to Columbus, Ohio, plane to Waynoka,\n",
      "Oklahoma, sleeper again to Clovis, New Mexico, and a plane the rest of the way.)\n",
      "Babe Ruth had knocked out forty home runs so far in the season. As a best-\n",
      "seller, All Quiet on the Western Front held a commanding lead over Dodsworth.\n",
      "Women's dresses were all emphasizing a decidedly flat look without anyone's\n",
      "saying so. From Washington it was announced that Harry F. Sinclair, then in the\n",
      "District of Columbia jail for being in contempt of the Senate during the Teapot\n",
      "Dome investigations, would henceforth be more closely confined. Previously he\n",
      "had been going daily by automobile to the office of the jail physician whom he\n",
      "was serving as a \"pharmaceutical assistant.\" Earlier in the year Sinclair's\n",
      "stock market operations were on a huge scale, and they were later the subject of\n",
      "detailed investigation. It was never shown whether the Washington sojourn\n",
      "involved any important interruption. It seems most unlikely that it did. Mr.\n",
      "Sinclair was one of the most resourceful and resilent entrepreneurs of his\n",
      "generation. On September 3 sales on the New York Stock Exchange were 4,438,910\n",
      "shares; call money was 9 per cent all day; the rate at the banks on prime\n",
      "commercial paper was 6½ per cent; the rediscount rate at the New York Federal\n",
      "Reserve Bank was 6 per cent. The market was strong with what the market\n",
      "reporters called a good undertone.American Tel and Tel reached 304 that day.\n",
      "U.S. Steel reached 262; General Electric was 396; J. I. Case, 350; New York\n",
      "Central, 256; Radio Corporation of America, adjusted for earlier split-ups and\n",
      "still not having paid a dividend, was 505. The brokers' loan figures of the\n",
      "Federal Reserve, when they were released, also showed another huge\n",
      "increase—$137,000,000 in one week. The New York banks were also borrowing\n",
      "heavily from the Federal Reserve to carry the speculative superstructure—their\n",
      "borrowings during the week increased by $64,000,000. In August the flow of gold\n",
      "to New York from abroad had continued large. Yet the new month seemed to be\n",
      "opening well. There were several expressions of confidence. On September 3, by\n",
      "common consent, the great bull market of the nineteen-twenties came to an end.\n",
      "Economics, as always, vouchsafes us few dramatic turning points. Its events are\n",
      "invariably fuzzy or even indeterminate. On some days that followed—a few\n",
      "only—some averages were actually higher. However, never again did the market\n",
      "manifest its old confidence. The later peaks were not peaks but brief\n",
      "interruptions of a downward trend.On September 4, the tone of the market was\n",
      "still good, and then on September 5 came a break. The Times industrials dropped\n",
      "10 points, and many individual stocks much more. The blue chips held up fairly\n",
      "well, although Steel went from 255 to 246, while Westinghouse lost 7 points and\n",
      "Tel and Tel 6. Volume mounted sharply as people sought to unload, and 5,565,280\n",
      "shares were traded on the New York Stock Exchange.The immediate cause of the\n",
      "break was clear—and interesting. Speaking before his Annual National Business\n",
      "Conference on September 5, Roger Babson observed, \"Sooner or later a crash is\n",
      "coming, and it may be terrific.\" He suggested that what had happened in Florida\n",
      "would now happen in Wall Street, and with customary precision stated that the\n",
      "(Dow-Jones) market averages would probably drop 60–80 points. In a burst of\n",
      "cheer he concluded that \"factories will shut down ... men will be thrown out of\n",
      "work ... the vicious circle will get in full swing and the result will be a\n",
      "serious business depression.\"21 This was not exactly reassuring. Yet it was a\n",
      "problem why the market suddenly should pay attention to Babson. As many hastened\n",
      "to say, he had made many predictions before, and they had not affected prices\n",
      "much one way or another. Moreover, Babson was not a man who inspired confidence\n",
      "as a prophet in the manner of Irving Fisher or the Harvard Economic Society. As\n",
      "an educator, philosopher, theologian, statistician, forecaster, economist, and\n",
      "friend of the law of gravity, he had sometimes been thought to spread himself\n",
      "too thin. The methods by which he reached his conclusions were a problem. They\n",
      "involved a hocus-pocus of lines and areas on a chart. Intuition, and possibly\n",
      "even mysticism, played a part. Those who employed rational, objective, and\n",
      "scientific methods were naturally uneasy about Babson, although their methods\n",
      "failed to foretell the crash. In these matters, as often in our culture, it is\n",
      "far, far better to be wrong in a respectable way than to be right for the wrong\n",
      "reasons.Wall Street was not at loss as to what to do about Babson. It promptly\n",
      "and soundly denounced him. Barron's, in an editorial on September 9, referred to\n",
      "him with heavy irony as the \"sage of Wellesley\" and said he should not be taken\n",
      "seriously by anyone acquainted with the \"notorious inaccuracy\" of his past\n",
      "statements. The stock exchange house of Hornblower and Weeks sternly told its\n",
      "customers: \"We would not be stampeded into selling stocks because of a\n",
      "gratuitous forecast of a bad break in the market by a well-known\n",
      "statistician.\"22 Irving Fisher also took issue. He noted that dividends were\n",
      "rising, that the suspicion of common stocks was receding, and that investment\n",
      "trusts now offered the investor \"wide and well-managed diversification.\" His own\n",
      "conclusion was: \"There may be a recession in stock prices, but not anything in\n",
      "the nature of a crash.\" 23 Developing a slightly different theme, a Boston\n",
      "investment trust told the public that it should be prepared for slight setbacks,\n",
      "but it should realize that they would soon pass. In large advertisements it\n",
      "pointed out that \"When temporary breaks come, indentations in the ever ascending\n",
      "curve of American prosperity, individual stocks, even of the most successful\n",
      "companies, go down with the general list...\" However, it also stated on its own\n",
      "behalf that \"Incorporated Investors lands on a cushion.\" The Babson Break, as it\n",
      "was promptly called, came on a Thursday. The market rallied on Friday and was\n",
      "firm on Saturday. People seemed to be over their fear. It looked as though the\n",
      "ever ascending curve would start up again, as so often before and Mr. Babson\n",
      "notwithstanding. Then on the following week—the week of September 9—prices were\n",
      "again ragged. On Monday, the Times, with the caution born of much premature\n",
      "pessimism, hinted that the end had come and added, \"It is a well-known\n",
      "characteristic of 'boom times' that the idea of their being terminated in the\n",
      "old, unpleasant way is rarely recognized as possible.\" On Wednesday, in a fine\n",
      "example of market prose, the Wall Street Journal observed that \"price movements\n",
      "in the main body of stocks yesterday continued to display the characteristics of\n",
      "a major advance temporarily halted for technical readjustment.\"The unevenness\n",
      "continued. On some days the market was strong; on others it was weak. The\n",
      "direction was slightly, erratically, but, viewed in retrospect, quite definitely\n",
      "down. New investment trusts were still being formed; more speculators were\n",
      "flocking to the market, and the volume of brokers' loans continued sharply up.\n",
      "The end had come, but it was not yet in sight. Perhaps this was as well. The\n",
      "last moments of life should be cherished, as Wall Street had been told. On\n",
      "September 11, in keeping with a regular practice, The Wall Street Journal\n",
      "printed its thought for the day. It was from Mark Twain. Don't part with your\n",
      "illusions; when they are gone you may still exist, but you have ceased to live.\n",
      "CHAPTER VThe CrashACCORDING TO the accepted view of events, by the autumn of\n",
      "1929 the economy was well into a depression. In June the indexes of industrial\n",
      "and of factory production both reached a peak and turned down. By October, the\n",
      "Federal Reserve index of industrial production stood at 117 as compared with 126\n",
      "four months earlier. Steel production declined from June on; in October freight-\n",
      "car loadings fell. Home-building, a most mercurial industry, had been falling\n",
      "for several years, and it slumped still farther in 1929. Finally, down came the\n",
      "stock market. A penetrating student of the economic behavior of this period has\n",
      "said that the market slump, \"reflected, in the main, the change which was\n",
      "already apparent in the industrial situation.\" 1Thus viewed, the stock market is\n",
      "but a mirror which, perhaps as in this instance, somewhat belatedly, provides an\n",
      "image of the underlying or fundamental economic situation. Cause and effect run\n",
      "from the economy to the stock market, never the reverse. In 1929 the economy was\n",
      "headed for trouble. Eventually that trouble was violently reflected in Wall\n",
      "Street.In 1929 there were good, or at least strategic, reasons for this view,\n",
      "and it is easy to understand why it has become high doctrine. In Wall Street, as\n",
      "elsewhere in 1929, few people wanted a bad depression. In Wall Street, as\n",
      "elsewhere, there is deep faith in the power of incantation. When the market fell\n",
      "many Wall Street citizens immediately sensed the real danger, which was that\n",
      "income and employment—prosperity in general—would be adversely affected. This\n",
      "had to be prevented. Preventive incantation required that as many important\n",
      "people as possible repeat as firmly as they could that it wouldn't happen. This\n",
      "they did. They explained how the stock market was merely the froth and that the\n",
      "real substance of economic life rested in production, employment, and spending,\n",
      "all of which would remain unaffected. No one knew for sure that this was so. As\n",
      "an instrument of economic policy, incantation does not permit of minor doubts or\n",
      "scruples. In the later years of depression it was important to continue\n",
      "emphasizing the unimportance of the stock market. The depression was an\n",
      "exceptionally disagreeable experience. Wall Street has not always been a\n",
      "cherished symbol in our national life. In some of the devout regions of the\n",
      "nation, those who speculate in stocks—the even more opprobrious term gamblers is\n",
      "used—are not counted the greatest moral adornments of our society. Any\n",
      "explanation of the depression which attributed importance to the market collapse\n",
      "would accordingly have been taken very seriously, and it would have meant\n",
      "serious trouble for Wall Street. Wall Street, no doubt, would have survived, but\n",
      "there would have been scars. We should be clear that no deliberate conspiracy\n",
      "existed to minimize the consequences of the Wall Street crash for the economy.\n",
      "Rather, it merely appeared to everyone with an instinct for conservative\n",
      "survival that Wall Street had better be kept out of it. It was vulnerable.In\n",
      "fact, any satisfactory explanation of the events of the autumn of 1929 and\n",
      "thereafter must accord a dignified role to the speculative boom and ensuing\n",
      "collapse. Until September or October of 1929 the decline in economic activity\n",
      "was very modest. As I shall argue later, until after the market crash one could\n",
      "reasonably assume that this downward movement might soon reverse itself, as a\n",
      "similar movement had reversed itself in 1927 or did subsequently in 1949. There\n",
      "were no reasons for expecting disaster. No one could foresee that production,\n",
      "prices, incomes, and all other indicators would continue to shrink through three\n",
      "long and dismal years. Only after the market crash were there plausible grounds\n",
      "to suppose that things might now for a long while get a lot worse. From the\n",
      "foregoing it follows that the crash did not come—as some have suggested—because\n",
      "the market suddenly became aware that a serious depression was in the offing. A\n",
      "depression, serious or otherwise, could not be foreseen when the market fell.\n",
      "There is still the possibility that the downturn in the indexes frightened the\n",
      "speculators, led them to unload their stocks, and so punctured a bubble that had\n",
      "in any case to be punctured one day. This is more plausible. Some people who\n",
      "were watching the indexes may have been persuaded by this intelligence to sell,\n",
      "and others may then have been encouraged to follow. This is not very important,\n",
      "for it is in the nature of a speculative boom that almost anything can collapse\n",
      "it. Any serious shock to confidence can cause sales by those speculators who\n",
      "have always hoped to get out before the final collapse, but after all possible\n",
      "gains from rising prices have been reaped. Their pessimism will infect those\n",
      "simpler souls who had thought the market might go up forever but who now will\n",
      "change their minds and sell. Soon there will be margin calls, and still others\n",
      "will be forced to sell. So the bubble breaks.Along with the downturn of the\n",
      "indexes Wall Street has always attributed importance to two other events in the\n",
      "pricking of the bubble. In England on September 20, 1929, the enterprises of\n",
      "Clarence Hatry suddenly collapsed. Hatry was one of those curiously un-English\n",
      "figures with whom the English periodically find themselves unable to cope.\n",
      "Although his earlier financial history had been anything but reassuring, Hatry\n",
      "in the twenties had built up an industrial and financial empire of truly\n",
      "impressive proportions. The nucleus, all the more remarkably, was a line of\n",
      "coin-in-the-slot vending and automatic photograph machines. From these\n",
      "unprepossessing enterprises he had marched on into investment trusts and high\n",
      "finance. His expansion owed much to the issuance of unauthorized stock, the\n",
      "increase of assets by the forging of stock certificates, and other equally\n",
      "informal financing. In the lore of 1929, the unmasking of Hatry in London is\n",
      "supposed to have struck a sharp blow to confidence in New York.2 Ranking with\n",
      "Hatry in this lore was the refusal on October 11 of the Massachusetts Department\n",
      "of Public Utilities to allow Boston Edison to split its stock four to one. As\n",
      "the company argued, such split-ups were much in fashion. To avoid going along\n",
      "was to risk being considered back in the corporate gaslight era. The refusal was\n",
      "unprecedented. Moreover, the Department added insult to injury by announcing an\n",
      "investigation of the company's rates and by suggesting that the present value of\n",
      "the stock, \"due to the action of speculators,\" had reached a level where \"no\n",
      "one, in our judgment ... on the basis of its earnings, would find it to his\n",
      "advantage to buy it.\"These were uncouth words. They could have been important\n",
      "as, conceivably, could have been the exposure of Clarence Hatry. But it could\n",
      "also be that the inherently unstable equilibrium was shattered simply by a\n",
      "spontaneous decision to get out. On September 22, the financial pages of the New\n",
      "York papers carried an advertisement of an investment service with the arresting\n",
      "headline, OVERSTAYING A BULL MARKET. Its message read as follows: \"Most\n",
      "investors make money in a bull market, only to lose all profits made—and\n",
      "sometimes more—in the readjustment that inevitably follows.\" Instead of the\n",
      "downturn in the Federal Reserve industrial index, the exposure of Hatry, or the\n",
      "unnatural obstinacy of the Massachusetts Department of Public Utilities, it\n",
      "could have been such thoughts stirring first in dozens and then in hundreds, and\n",
      "finally in thousands of breasts which finally brought an end to the boom. What\n",
      "first stirred these doubts we do not know, but neither is it very important that\n",
      "we know.         IIConfidence did not disintegrate at once. As noted, through\n",
      "September and into October, although the trend of the market was generally down,\n",
      "good days came with the bad. Volume was high. On the New York Stock Exchange\n",
      "sales were nearly always above four million, and frequently above five. In\n",
      "September new issues appeared in even greater volume than in August, and they\n",
      "regularly commanded a premium over the offering price. On September 20 the Times\n",
      "noted that the stock of the recently launched Lehman Corporation which had been\n",
      "offered at $104 had sold the day before at $136. (In the case of this well-\n",
      "managed investment trust the public enthusiasm was not entirely misguided.)\n",
      "During September brokers' loans increased by nearly $670 million, by far the\n",
      "largest increase of any month to date. This showed that speculative zeal had not\n",
      "diminished.Other signs indicated that the gods of the New Era were still in\n",
      "their temples. In its October 12 issue, the Saturday Evening Post had a lead\n",
      "story by Isaac F. Marcosson on Ivar Kreuger. This was a scoop, for Kreuger had\n",
      "previously been inaccessible to journalists. \"Kreuger,\" Marcosson observed,\n",
      "\"like Hoover, is an engineer. He has consistently applied engineer precision to\n",
      "the welding of his far-flung industry.\" And this was not the only resemblance.\n",
      "\"Like Hoover,\" the author added, \"Kreuger rules through pure reason.\" In the\n",
      "interview Kreuger was remarkably candid on one point. He told Mr. Marcosson:\n",
      "\"Whatever success I have had may perhaps be attributable to three things: One is\n",
      "silence, the second is more silence, while the third is still more silence.\"\n",
      "This was so. Two and a half years later Kreuger committed suicide in his Paris\n",
      "apartment, and shortly thereafter it was discovered that his aversion to\n",
      "divulging information, especially if accurate, had kept even his most intimate\n",
      "acquaintances in ignorance of the greatest fraud in history. His American\n",
      "underwriters, the eminently respectable firm of Lee, Higginson and Company of\n",
      "Boston, had heard nothing and knew nothing. One of the members of the firm,\n",
      "Donald Durant, was a member of the board of directors of the Kreuger\n",
      "enterprises. He had never attended a directors' meeting, and it is certain that\n",
      "he would have been no wiser had he done so.During the last weeks of October,\n",
      "Time Magazine, young and not yet omniscient, also featured Kreuger on its\n",
      "cover—\"a great admirer of Cecil Rhodes.\" Then a week later, as though to\n",
      "emphasize its faith in the New Era, it went on to Samuel Insull. (A fortnight\n",
      "after that, its youthful illusions shattered, the weekly newsmagazine gave the\n",
      "place of historic honor to Warden Lawes of Sing Sing.) In these same Indian\n",
      "summer days, The Wall Street Journal took notice of the official announcement\n",
      "that Andrew Mellon would remain in the cabinet at least until 1933 (there had\n",
      "been rumors that he might resign) and observed: \"Optimism again prevails ... the\n",
      "announcement ... did more to restore confidence than anything else.\" In Germany\n",
      "Charles E. Mitchell announced that the \"industrial condition of the United\n",
      "States is absolutely sound,\" that too much attention was being paid to brokers'\n",
      "loans, and that \"nothing can arrest the upward movement.\" On October 15, as he\n",
      "sailed for home, he enlarged on the point: \"The markets generally are now in a\n",
      "healthy condition ... values have a sound basis in the general prosperity of our\n",
      "country.\" That same evening Professor Irving Fisher made his historic\n",
      "announcement about the permanently high plateau and added, \"I expect to see the\n",
      "stock market a good deal higher than it is today within a few months.\" Indeed,\n",
      "the only disturbing thing, in these October days, was the fairly steady downward\n",
      "drift in the market.         IIIOn Saturday, October 19, Washington dispatches\n",
      "reported that Secretary of Commerce Lamont was having trouble finding the\n",
      "$100,000 in public funds that would be required to pay the upkeep of the yacht\n",
      "Corsair which J. P. Morgan had just given the government. (Morgan's deprivation\n",
      "was not extreme: a new $3,000,000 Corsair was being readied at Bath, Maine.)\n",
      "There were other and more compelling indications of an unaccustomed stringency.\n",
      "The papers told of a very weak market the day before—there were heavy declines\n",
      "on late trading, and the Times industrial average had dropped about 7 points.\n",
      "Steel had lost 7 points; General Electric, Westinghouse, and Montgomery Ward all\n",
      "lost 6. Meanwhile, that day's market was behaving very badly. In the second\n",
      "heaviest Saturday's trading in history, 3,488,100 shares were changing hands. At\n",
      "the close the Times industrials were down 12 points. The blue chips were\n",
      "seriously off, and speculative favorites had gone into a nosedive. J. I. Case,\n",
      "for example, had fallen a full 40 points. On Sunday the market was front-page\n",
      "news—the Times headline read, \"Stocks driven down as wave of selling engulfs the\n",
      "market,\" and the financial editor next day reported for perhaps the tenth time\n",
      "that the end had come. (He had learned, however, to hedge. \"For the time at any\n",
      "rate,\" he said, \"Wall Street seemed to see the reality of things.\") No immediate\n",
      "explanation of the break was forthcoming. The Federal Reserve had long been\n",
      "quiet. Babson had said nothing new. Hatry and the Massachusetts Department of\n",
      "Public Utilities were from a week to a month in the past. They became\n",
      "explanations only later.The papers that Sunday carried three comments which were\n",
      "to become familiar in the days that followed. After Saturday's trading, it was\n",
      "noted, quite a few margin calls went out. This meant that the value of stock\n",
      "which the recipients held on margin had declined to the point where it was no\n",
      "longer sufficient collateral for the loan that had paid for it. The speculator\n",
      "was being asked for more cash.The other two observations were more reassuring.\n",
      "The papers agreed, and this was also the informed view on Wall Street, that the\n",
      "worst was over. And it was predicted that on the following day the market would\n",
      "begin to receive organized support. Weakness, should it appear, would be\n",
      "tolerated no longer.Never was there a phrase with more magic than \"organized\n",
      "support.\" Almost immediately it was on every tongue and in every news story\n",
      "about the market. Organized support meant that powerful people would organize to\n",
      "keep prices of stocks at a reasonable level. Opinions differed as to who would\n",
      "organize this support. Some had in mind the big operators like Cutten, Durant,\n",
      "and Raskob. They, of all people, couldn't afford a collapse. Some thought of the\n",
      "bankers—Charles Mitchell had acted once before, and certainly if things got bad\n",
      "he would act again. Some had in mind the investment trusts. They held huge\n",
      "portfolios of common stocks, and obviously they could not afford to have them\n",
      "become cheap. Also, they had cash. So if stocks did become cheap the investment\n",
      "trusts would be in the market picking up bargains. This would mean that the\n",
      "bargains wouldn't last. With so many people wanting to avoid a further fall, a\n",
      "further fall would clearly be avoided.In the ensuing weeks the Sabbath pause had\n",
      "a marked tendency to breed uneasiness and doubts and pessimism and a decision to\n",
      "get out on Monday. This, it seems certain, was what happened on Sunday, October\n",
      "20.         IVMonday, October 21, was a very poor day. Sales totaled 6,091,870,\n",
      "the third greatest volume in history, and some tens of thousands who were\n",
      "watching the market throughout the country made a disturbing discovery. There\n",
      "was no way of telling what was happening. Previously on big days of the bull\n",
      "market the ticker had often fallen behind, and one didn't discover until well\n",
      "after the market closed how much richer he had become. But the experience with a\n",
      "falling market had been much more limited. Not since March had the ticker fallen\n",
      "seriously behind on declining values. Many now learned for the first time that\n",
      "they could be ruined, totally and forever, and not even know it. And if they\n",
      "were not ruined there was a strong tendency to imagine it. From the opening on\n",
      "the 21st the ticker lagged, and by noon it was an hour late. Not until an hour\n",
      "and forty minutes after the close of the market did it record the last\n",
      "transaction. Every ten minutes prices of selected stocks were printed on the\n",
      "bond ticker, but the wide divergence between these and the prices on the tape\n",
      "only added to the uneasiness—and to the growing conviction that it might be best\n",
      "to sell. Things though bad were still not hopeless. Toward the end of Monday's\n",
      "trading the market rallied and final prices were above the lows for the day. The\n",
      "net losses were considerably less than on Saturday. Tuesday brought a somewhat\n",
      "shaky gain. As so often before, the market seemed to be showing its ability to\n",
      "come back. People got ready to record the experience as merely another setback\n",
      "of which there had been so many previously.In doing so they were helped by the\n",
      "two men who now were recognized as Wall Street's official prophets. On Monday in\n",
      "New York, Professor Fisher said that the decline had represented only a \"shaking\n",
      "out of the lunatic fringe.\" He went on to explain why he felt that the prices of\n",
      "stocks during the boom had not caught up with their real value and would go\n",
      "higher. Among other things, the market had not yet reflected the beneficent\n",
      "effects of prohibition which had made the American worker \"more productive and\n",
      "dependable.\"On Tuesday, Charles E. Mitchell dropped anchor in New York with the\n",
      "observation that \"the decline had gone too far.\" (Time and sundry congressional\n",
      "and court proceedings were to show that Mr. Mitchell had strong personal reasons\n",
      "for feeling that way.) He added that conditions were \"fundamentally sound,\" said\n",
      "again that too much attention had been paid to the large volume of brokers'\n",
      "loans, and concluded that the situation was one which would correct itself if\n",
      "left alone. However, another jarring suggestion came from Babson. He recommended\n",
      "selling stocks and buying gold.By Wednesday, October 23, the effect of this\n",
      "cheer was somehow dissipated. Instead of further gains there were heavy losses.\n",
      "The opening was quiet enough, but toward midmorning motor accessory stocks were\n",
      "sold heavily, and volume began to increase throughout the list. The last hour\n",
      "was quite phenomenal—2,600,000 shares changed hands at rapidly declining prices.\n",
      "The Times industrial average for the day dropped from 415 to 384, giving up all\n",
      "of its gains since the end of the previous June. Tel and Tel lost 15 points;\n",
      "General Electric, 20; Westinghouse, 25; and J. I. Case, another 46. Again the\n",
      "ticker was far behind, and to add to the uncertainty an ice storm in the Middle\n",
      "West caused widespread disruption of communications. That afternoon and evening\n",
      "thousands of speculators decided to get out while—as they mistakenly\n",
      "supposed—the getting was good. Other thousands were told they had no choice but\n",
      "to get out unless they posted more collateral, for as the day's business came to\n",
      "an end an unprecedented volume of margin calls went out. Speaking in Washington,\n",
      "even Professor Fisher was fractionally less optimistic. He told a meeting of\n",
      "bankers that \"security values in most instances were not inflated.\" However, he\n",
      "did not weaken on the unrealized efficiencies of prohibition.The papers that\n",
      "night went to press with a souvenir of a fast departing era. Formidable\n",
      "advertisements announced subscription rights in a new offering of certificates\n",
      "in Aktiebolaget Kreuger and Toll at $23. There was also one bit of cheer. It was\n",
      "predicted that on the morrow the market would surely begin to receive \"organized\n",
      "support.\"         VThursday, October 24, is the first of the days which\n",
      "history—such as it is on the subject—identifies with the panic of 1929. Measured\n",
      "by disorder, fright, and confusion, it deserves to be so regarded. That day\n",
      "12,894,650 shares changed hands, many of them at prices which shattered the\n",
      "dreams and the hopes of those who had owned them. Of all the mysteries of the\n",
      "stock exchange there is none so impenetrable as why there should be a buyer for\n",
      "everyone who seeks to sell. October 24, 1929, showed that what is mysterious is\n",
      "not inevitable. Often there were no buyers, and only after wide vertical\n",
      "declines could anyone be induced to bid. The panic did not last all day. It was\n",
      "a phenomenon of the morning hours. The market opening itself was unspectacular,\n",
      "and for a while prices were firm. Volume, however, was very large, and soon\n",
      "prices began to sag. Once again the ticker dropped behind. Prices fell farther\n",
      "and faster, and the ticker lagged more and more. By eleven o'clock the market\n",
      "had degenerated into a wild, mad scramble to sell. In the crowded boardrooms\n",
      "across the country the ticker told of a frightful collapse. But the selected\n",
      "quotations coming in over the bond ticker also showed that current values were\n",
      "far below the ancient history of the tape. The uncertainty led more and more\n",
      "people to try to sell. Others, no longer able to respond to margin calls, were\n",
      "sold out. By eleven-thirty the market had surrendered to blind, relentless fear.\n",
      "This, indeed, was panic.Outside the Exchange in Broad Street a weird roar could\n",
      "be heard. A crowd gathered. Police Commissioner Grover Whalen became aware that\n",
      "something was happening and dispatched a special police detail to Wall Street to\n",
      "insure the peace. More people came and waited, though apparently no one knew for\n",
      "what. A workman appeared atop one of the high buildings to accomplish some\n",
      "repairs, and the multitude assumed he was a would-be suicide and waited\n",
      "impatiently for him to jump. Crowds also formed around the branch offices of\n",
      "brokerage firms throughout the city and, indeed, throughout the country. Word of\n",
      "what was happening, or what was thought to be happening, was passed out by those\n",
      "who were within sight of the board or the Trans-Lux. An observer thought that\n",
      "people's expressions showed \"not so much suffering as a sort of horrified\n",
      "incredulity.\" 3 Rumor after rumor swept Wall Street and these outlying wakes.\n",
      "Stocks were now selling for nothing. The Chicago and Buffalo Exchanges had\n",
      "closed. A suicide wave was in progress, and eleven well-known speculators had\n",
      "already killed themselves.At twelve-thirty the officials of the New York Stock\n",
      "Exchange closed the visitors gallery on the wild scenes below. One of the\n",
      "visitors who had just departed was showing his remarkable ability to be on hand\n",
      "with history. He was the former Chancellor of the Exchequer, Mr. Winston\n",
      "Churchill. It was he who in 1925 returned Britain to the gold standard and the\n",
      "overvalued pound. Accordingly, he was responsible for the strain which sent\n",
      "Montagu Norman to plead in New York for easier money, which caused credit to be\n",
      "eased at the fatal time, which, in this academy view, in turn caused the boom.\n",
      "Now Churchill, it could be imagined, was viewing his awful handiwork.There is no\n",
      "record of anyone's having reproached him. Economics was never his strong point,\n",
      "so (and wisely) it seems most unlikely that he reproached himself.         VIIn\n",
      "New York at least the panic was over by noon. At noon the organized support\n",
      "appeared. At twelve o'clock reporters learned that a meeting was convening at 23\n",
      "Wall Street at the offices of J. P. Morgan and Company. The word quickly passed\n",
      "as to who was there—Charles E. Mitchell, the Chairman of the Board of the\n",
      "National City Bank, Albert H. Wiggin, the Chairman of the Chase National Bank,\n",
      "William C. Potter, the President of the Guaranty Trust Company, Seward Prosser,\n",
      "the Chairman of the Bankers Trust Company, and the host, Thomas W. Lamont, the\n",
      "senior partner of Morgan's. According to legend, during the panic of 1907 the\n",
      "elder Morgan had brought to a halt the discussion of whether to save the\n",
      "tottering Trust Company of America by saying that the place to stop the panic\n",
      "was there. It was stopped. Now, twenty-two years later, that drama was being re-\n",
      "enacted. The elder Morgan was dead. His son was in Europe. But equally\n",
      "determined men were moving in. They were the nation's most powerful financiers.\n",
      "They had not yet been pilloried and maligned by New Dealers. The very news that\n",
      "they would act would release people from the fear to which they had\n",
      "surrendered.It did. A decision was quickly reached to pool resources to support\n",
      "the market.4 The meeting broke up, and Thomas Lamont met with reporters. His\n",
      "manner was described as serious, but his words were reassuring. In what\n",
      "Frederick Lewis Allen later called one of the most remarkable understatements of\n",
      "all time,5 he told the newspapermen, \"There has been a little distress selling\n",
      "on the Stock Exchange.\" He added that this was \"due to a technical condition of\n",
      "the market\" rather than any fundamental cause, and told the newsmen that things\n",
      "were \"susceptible to betterment.\" The bankers, he let it be known, had decided\n",
      "to better things. Word had already reached the floor of the Exchange that the\n",
      "bankers were meeting, and the news ticker had spread the magic word afield.\n",
      "Prices firmed at once and started to rise. Then at one-thirty Richard Whitney\n",
      "appeared on the floor and went to the post where steel was traded. Whitney was\n",
      "perhaps the best-known figure on the floor. He was one of the group of men of\n",
      "good background and appropriate education who, in that time, were expected to\n",
      "manage the affairs of the Exchange. Currently he was vice-president of the\n",
      "Exchange, but in the absence of E. H. H. Simmons in Hawaii he was serving as\n",
      "acting president. What was much more important at the moment, he was known as\n",
      "floor trader for Morgan's and, indeed, his older brother was a Morgan partner.As\n",
      "he made his way through the teeming crowd, Whitney appeared debonair and self-\n",
      "confident—some later described his manner as jaunty. (His own firm dealt largely\n",
      "in bonds, so it is improbable that he had been much involved in the turmoil of\n",
      "the morning.) At the Steel post he bid 205 for 10,000 shares. This was the price\n",
      "of the last sale, and the current bids were several points lower. In an\n",
      "operation that was totally devoid of normal commercial reticence, he got 200\n",
      "shares and then left the rest of the order with the specialist. He continued on\n",
      "his way, placing similar orders for fifteen or twenty other stocks.This was it.\n",
      "The bankers, obviously, had moved in. The effect was electric. Fear vanished and\n",
      "gave way to concern lest the new advance be missed. Prices boomed upward.The\n",
      "bankers had, indeed, brought off a notable coup. Prices as they fell that\n",
      "morning kept crossing a large volume of stop-loss orders—orders calling for\n",
      "sales whenever a specified price was reached. Brokers had placed many of these\n",
      "orders for their own protection on the securities of customers who had not\n",
      "responded to calls for additional margin. Each of these stop-loss orders tripped\n",
      "more securities into the market and drove prices down farther. Each spasm of\n",
      "liquidation thus insured that another would follow. It was this literal chain\n",
      "reaction which the bankers checked, and they checked it decisively. In the\n",
      "closing hour, selling orders continuing to come in from across the country\n",
      "turned the market soft once more. Still, in its own way, the recovery on Black\n",
      "Thursday was as remarkable as the selling that made it so black. The Times\n",
      "industrials were off only 12 points, or a little more than a third of the loss\n",
      "of the previous day. Steel, the stock that Whitney had singled out to start the\n",
      "recovery, had opened that morning at 205½, a point or two above the previous\n",
      "close. At the lowest it was down to 193½ for a 12-point loss.6 Then it recovered\n",
      "to close at 206 for a surprising net gain of 2 points for this day. Montgomery\n",
      "Ward, which had opened at 83 and gone to 50, came back to 74. General Electric\n",
      "was at one point 32 points below its opening price and then came back 25 points.\n",
      "On the Curb, Goldman Sachs Trading Corporation opened at 81, dropped to 65, and\n",
      "then came back to 80. J. I. Case, maintaining a reputation for eccentric\n",
      "behavior that had brought much risk capital into the threshing machine business,\n",
      "made a net gain of 7 points for the day. Many had good reason to be grateful to\n",
      "the financial leaders of Wall Street.         VIINot everyone could be grateful\n",
      "to be sure. Across the country people were only dimly aware of the improvement.\n",
      "By early afternoon, when the market started up, the ticker was hours behind.\n",
      "Although the spot quotations on the bond ticker showed the improvement, the\n",
      "ticker itself continued to grind out the most dismal of news. And the news on\n",
      "the ticker was what counted. To many, many watchers it meant that they had been\n",
      "sold out and that their dream—in fact, their brief reality—of opulence had gone\n",
      "glimmering, together with home, car, furs, jewelry, and reputation. That the\n",
      "market, after breaking them, had recovered was the most chilling of comfort.It\n",
      "was eight and a half minutes past seven that night before the ticker finished\n",
      "recording the day's misfortunes. In the boardrooms speculators who had been sold\n",
      "out since morning sat silently watching the tape. The habit of months or years,\n",
      "however idle it had now become, could not be abandoned at once. Then, as the\n",
      "final trades were registered, sorrowfully or grimly, according to their nature,\n",
      "they made their way out into the gathering night.In Wall Street itself lights\n",
      "blazed from every office as clerks struggled to come abreast of the day's\n",
      "business. Messengers and boardroom boys, caught up in the excitement and\n",
      "untroubled by losses, went skylarking through the streets until the police\n",
      "arrived to quell them. Representatives of thirty-five of the largest wire houses\n",
      "assembled at the offices of Hornblower and Weeks and told the press on departing\n",
      "that the market was \"fundamentally sound\" and \"technically in better condition\n",
      "than it has been in months.\" It was the unanimous view of those present that the\n",
      "worst had passed. The host firm dispatched a market letter which stated that\n",
      "\"commencing with today's trading the market should start laying the foundation\n",
      "for the constructive advance which we believe will characterize 1930.\" Charles\n",
      "E. Mitchell announced that the trouble was \"purely technical\" and that\n",
      "\"fundamentals remained unimpaired.\" Senator Carter Glass said the trouble was\n",
      "due largely to Charles E. Mitchell. Senator Wilson of Indiana attributed the\n",
      "crash to Democratic resistance to a higher tariff.         VIIIOn Friday and\n",
      "Saturday trading continued heavy—just under six million on Friday and over two\n",
      "million at the short session on Saturday. Prices, on the whole, were steady—the\n",
      "averages were a trifle up on Friday but slid off on Saturday. It was thought\n",
      "that the bankers were able to dispose of most of the securities they had\n",
      "acquired while shoring up the market on Thursday. Not only were things better,\n",
      "but everyone was clear as to who had made them so. The bankers had shown both\n",
      "their coin-age and their power, and the people applauded warmly and generously.\n",
      "The financial community, the Times said, now felt \"secure in the knowledge that\n",
      "the most powerful banks in the country stood ready to prevent a recurrence [of\n",
      "panic].\" As a result it had \"relaxed its anxiety.\"Perhaps never before or since\n",
      "have so many people taken the measure of economic prospects and found them so\n",
      "favorable as in the two days following the Thursday disaster. The optimism even\n",
      "included a note of self-congratulation. Colonel Ayres in Cleveland thought that\n",
      "no other country could have come through such a bad crash so well. Others\n",
      "pointed out that the prospects for business were good and that the stock market\n",
      "debacle would not make them any less favorable. No one knew, but it cannot be\n",
      "stressed too frequently, that for effective incantation knowledge is neither\n",
      "necessary nor assumed. Eugene M. Stevens, the President of the Continental\n",
      "Illinois Bank, said, \"There is nothing in the business situation to justify any\n",
      "nervousness.\" Walter Teagle said there had been no \"fundamental change\" in the\n",
      "oil business to justify concern; Charles M. Schwab said that the steel business\n",
      "had been making \"fundamental progress\" toward stability and added that this\n",
      "\"fundamentally sound condition\" was responsible for the prosperity of the\n",
      "industry; Samuel Vauclain, Chairman of the Baldwin Locomotive Works, declared\n",
      "that \"fundamentals are sound\"; President Hoover said that \"the fundamental\n",
      "business of the country, that is production and distribution of commodities, is\n",
      "on a sound and prosperous basis.\" President Hoover was asked to say something\n",
      "more specific about the market—for example, that stocks were now cheap—but he\n",
      "refused.7Many others joined in. Howard C. Hopson, the head of Associated Gas and\n",
      "Electric, omitted the standard reference to fundamentals and thought it was\n",
      "\"undoubtedly beneficial to the business interests of the country to have the\n",
      "gambling type of speculator eliminated.\" (Mr. Hopson, himself a speculator,\n",
      "although more of the sure-thing type, was also eliminated in due course.) A\n",
      "Boston investment trust took space in The Wall Street Journal to say, \"S-T-E-A-\n",
      "D-Y Everybody! Calm thinking is in order. Heed the words of America's greatest\n",
      "bankers.\" A single dissonant note, though great in portent, went unnoticed.\n",
      "Speaking in Poughkeepsie, Governor Franklin D. Roosevelt criticized the \"fever\n",
      "of speculation.\" On Sunday there were sermons suggesting that a certain measure\n",
      "of divine retribution had been visited on the Republic and that it had not been\n",
      "entirely unmerited. People had lost sight of spiritual values in their single-\n",
      "minded pursuit of riches. Now they had had their lesson.Almost everyone believed\n",
      "that the heavenly knuckle-rapping was over and that speculation could be now\n",
      "resumed in earnest. The papers were full of the prospects for next week's\n",
      "market.Stocks, it was agreed, were again cheap and accordingly there would be a\n",
      "heavy rush to buy. Numerous stories from the brokerage houses, some of them\n",
      "possibly inspired, told of a fabulous volume of buying orders which was piling\n",
      "up in anticipation of the opening of the market. In a concerted advertising\n",
      "campaign in Monday's papers, stock market firms urged the wisdom of picking up\n",
      "these bargains promptly. \"We believe,\" said one house, \"that the investor who\n",
      "purchases securities at this time with the discrimination that is always a\n",
      "condition of prudent investing, may do so with utmost confidence.\" On Monday the\n",
      "real disaster began.         CHAPTER VIThings Become More SeriousIN THE AUTUMN\n",
      "of 1929 the New York Stock Exchange, under roughly its present constitution, was\n",
      "112 years old. During this lifetime it had seen some difficult days. On\n",
      "September 18, 1873, the firm of Jay Cooke and Company failed, and, as a more or\n",
      "less direct result, so did fifty-seven other stock exchange firms in the next\n",
      "few weeks. On October 23, 1907, call money rates reached 125 per cent in the\n",
      "panic of that year. On September 16, 1920—the autumn months are the off season\n",
      "in Wall Street—a bomb exploded in front of Morgan's next door, killing thirty\n",
      "people and injuring a hundred more.A common feature of all these earlier\n",
      "troubles was that having happened they were over. The worst was reasonably\n",
      "recognizable as such. The singular feature of the great crash of 1929 was that\n",
      "the worst continued to worsen. What looked one day like the end proved on the\n",
      "next day to have been only the beginning. Nothing could have been more\n",
      "ingeniously designed to maximize the suffering, and also to insure that as few\n",
      "as possible escaped the common misfortune. The fortunate speculator who had\n",
      "funds to answer the first margin call presently got another and equally urgent\n",
      "one, and if he met that there would still be another. In the end all the money\n",
      "he had was extracted from him and lost. The man with the smart money, who was\n",
      "safely out of the market when the first crash came, naturally went back in to\n",
      "pick up bargains. (Not only were a recorded 12,894,650 shares sold on October\n",
      "24; precisely the same number were bought.) The bargains then suffered a ruinous\n",
      "fall. Even the man who waited out all of October and all of November, who saw\n",
      "the volume of trading return to normal and saw Wall Street become as placid as a\n",
      "produce market, and who then bought common stocks would see their value drop to\n",
      "a third or a fourth of the purchase price in the next twenty-four months. The\n",
      "Coolidge bull market was a remarkable phenomenon. The ruthlessness of its\n",
      "liquidation was, in its own way, equally remarkable.         IIMonday, October\n",
      "28, was the first day on which this process of climax and anticlimax ad\n",
      "infinitum began to reveal itself. It was another terrible day. Volume was huge,\n",
      "although below the previous Thursday—nine and a quarter million shares as\n",
      "compared with nearly thirteen. But the losses were far more severe. The Times\n",
      "industrials were down 49 points for the day. General Electric was off 48;\n",
      "Westinghouse, 34; Tel and Tel, 34. Steel went down 18 points. Indeed, the\n",
      "decline on this one day was greater than that of all the preceding week of\n",
      "panic. Once again a late ticker left everyone in ignorance of what was\n",
      "happening, save that it was bad.On this day there was no recovery. At one-ten\n",
      "Charles E. Mitchell was observed going into Morgan's, and the news ticker\n",
      "carried the magic word. Steel rallied and went from 194 to 198. But Richard\n",
      "Whitney did not materialize. It seems probable in light of later knowledge that\n",
      "Mitchell was on the way to float a personal loan. The market weakened again, and\n",
      "in the last hour a phenomenal three million shares—a big day's business before\n",
      "and ever since—changed hands at rapidly falling prices. At four-thirty in the\n",
      "afternoon the bankers assembled once more at Morgan's, and they remained in\n",
      "session until six-thirty. They were described as taking a philosophical\n",
      "attitude, and they told the press that the situation \"retained hopeful\n",
      "features,\" although these were not specified. But the statement they released\n",
      "after the meeting made clear what had been discussed for the two hours. It was\n",
      "no part of the bankers' purpose, the statement said, to maintain any particular\n",
      "level of prices or to protect anyone's profit. Rather the aim was to have an\n",
      "orderly market, one in which offers would be met by bids at some price. The\n",
      "bankers were only concerned that \"air holes,\" as Mr. Lamont had dubbed them, did\n",
      "not appear.Like many lesser men, Mr. Lamont and his colleagues had suddenly\n",
      "found themselves overcommitted on a falling market. The time had come to go\n",
      "short on promises. Support, organized or otherwise, could not contend with the\n",
      "overwhelming, pathological desire to sell. The meeting had considered how to\n",
      "liquidate the commitment to support the market without adding to the public\n",
      "perturbation.The formula that was found was a chilling one. On Thursday, Whitney\n",
      "had supported prices and protected profits—or stopped losses. This was what\n",
      "people wanted. To the man who held stock on margin, disaster had only one face\n",
      "and that was falling prices. But now prices were to be allowed to fall. The\n",
      "speculator's only comfort, henceforth, was that his ruin would be accomplished\n",
      "in an orderly and becoming manner.There were no recriminations at the time. Our\n",
      "political life favors the extremes of speech; the man who is gifted in the arts\n",
      "of abuse is bound to be a notable, if not always a great figure. In business\n",
      "things are different. Here we are surprisingly gentle and forbearing. Even\n",
      "preposterous claims or excuses are normally taken, at least for all public\n",
      "purposes, at their face value. On the evening of the 28th no one any longer\n",
      "could feel \"secure in the knowledge that the most powerful banks stood ready to\n",
      "prevent a recurrence\" of panic. The market had reasserted itself as an\n",
      "impersonal force beyond the power of any person to control, and, while this is\n",
      "the way markets are supposed to be, it was horrible. But no one assailed the\n",
      "bankers for letting the people down. There was even some talk that on the next\n",
      "day the market might receive organized support.         IIITuesday, October 29,\n",
      "was the most devastating day in the history of the New York stock market, and it\n",
      "may have been the most devastating day in the history of markets. It combined\n",
      "all of the bad features of all of the bad days before. Volume was immensely\n",
      "greater than on Black Thursday; the drop in prices was almost as great as on\n",
      "Monday. Uncertainty and alarm were as great as on either.Selling began as soon\n",
      "as the market opened and in huge volume. Great blocks of stock were offered for\n",
      "what they would bring; in the first half hour sales were at a 33,000,000-a-day\n",
      "rate. The air holes, which the bankers were to close, opened wide. Repeatedly\n",
      "and in many issues there was a plethora of selling orders and no buyers at all.\n",
      "The stock of White Sewing Machine Company, which had reached a high of 48 in the\n",
      "months preceding, had closed at 11 the night before. During the day\n",
      "someone—according to Frederick Lewis Allen it was thought to have been a bright\n",
      "messenger boy for the Exchange—had the happy idea of entering a bid for a block\n",
      "of stock at a dollar a share. In the absence of any other bid he got it.1 Once\n",
      "again, of course, the ticker lagged—at the close it was two and a half hours\n",
      "behind. By then, 16,410,030 sales had been recorded on the New York Stock\n",
      "Exchange—some certainly went unrecorded—or more than three times the number that\n",
      "was once considered a fabulously big day. The Times industrial averages were\n",
      "down 43 points, canceling all of the gains of the twelve wonderful months\n",
      "preceding. The losses would have been worse had there not been a closing rally.\n",
      "Thus Steel, for which Whitney had bid 205 on Thursday, reached 167 during the\n",
      "course of the day, although it rallied to 174 at the close. American Can opened\n",
      "at 130, dropped to 110, and rose to 120. Westinghouse opened at 131—on September\n",
      "3 it had closed at 286—and dropped to 100. Then it rallied to 126. But the worst\n",
      "thing that happened on this terrible day was to the investment trusts. Not only\n",
      "did they go down, but it became apparent that they could go practically to\n",
      "nothing. Goldman Sachs Trading Corporation had closed at 60 the night before.\n",
      "During the day it dropped to 35 and closed at that level, off by not far short\n",
      "of half. Blue Ridge, its offspring once removed, on which the magic of leverage\n",
      "was now working in reverse, did much worse. Early in September it had sold at\n",
      "24. By October 24 it was down to 12, but it resisted rather well the misfortunes\n",
      "of that day and the day following. On the morning of October 29 it opened at 10\n",
      "and promptly slipped to 3, giving up more than two thirds of its value. It\n",
      "recovered later but other investment trusts did less well; their stock couldn't\n",
      "be sold at all.The worst day on Wall Street came eventually to an end. Once\n",
      "again the lights blazed all night. Members of the Exchange, their employees, and\n",
      "the employees of the Stock Exchange by now were reaching the breaking point from\n",
      "strain and fatigue. In this condition they faced the task of recording and\n",
      "handling the greatest volume of transactions ever. All of this was without the\n",
      "previous certainty that things might get better. They might go on getting worse.\n",
      "In one house an employee fainted from exhaustion, was revived and put back to\n",
      "work again.         IVIn the first week the slaughter had been of the innocents.\n",
      "During this second week there is some evidence that it was the well-to-do and\n",
      "the wealthy who were being subjected to a leveling process comparable in\n",
      "magnitude and suddenness to that presided over a decade before by Lenin. The\n",
      "size of the blocks of stock which were offered suggested that big speculators\n",
      "were selling or being sold. Another indication came from the boardrooms. A week\n",
      "before they were crowded, now they were nearly empty. Those now in trouble had\n",
      "facilities for suffering in private.The bankers met twice on the 29th—at noon\n",
      "and again in the evening. There was no suggestion that they were philosophical.\n",
      "This was hardly remarkable because, during the day, an appalling rumor had swept\n",
      "the Exchange. It was that the bankers' pool, so far from stabilizing the market,\n",
      "was actually selling stocks! The prestige of the bankers had in truth been\n",
      "falling even more rapidly than the market. After the evening session, Mr. Lamont\n",
      "met the press with the disenchanting task of denying that they had been\n",
      "liquidating securities—or participating in a bear raid. After explaining again,\n",
      "somewhat redundantly in view of the day's events, that it was not the purpose of\n",
      "the bankers to maintain a particular level of prices, he concluded: \"The group\n",
      "has continued and will continue in a co-operative way to support the market and\n",
      "has not been a seller of stocks.\" In fact, as later intelligence revealed,\n",
      "Albert H. Wiggin of the Chase was personally short at the time to the time of\n",
      "some millions. His co-operative support, which if successful would have cost him\n",
      "heavily, must have had an interesting element of ambivalence. So ended the\n",
      "organized support. The phrase recurred during the next few days, but no one\n",
      "again saw in it any ground for hope. Few men ever lost position so rapidly as\n",
      "did the New York bankers in the five days from October 24 to October 29. The\n",
      "crash on October 24 was the signal for corporations and out-of-town banks, which\n",
      "had been luxuriating in the 10 per cent and more rate of interest, to recall\n",
      "their money from Wall Street. Between October 23 and October 30, as values fell\n",
      "and margin accounts were liquidated, the volume of brokers' loans fell by over a\n",
      "billion. But the corporations and the out-of-town banks responded to the\n",
      "horrifying news from New York—although, in fact, their funds were never\n",
      "seriously endangered—by calling home over two billions. The New York banks\n",
      "stepped into the gaping hole that was left by these summer financiers, and\n",
      "during that first week of crisis they increased their loans by about a billion.\n",
      "This was a bold step. Had the New York banks succumbed to the general fright, a\n",
      "money panic would have been added to the other woes. Stocks would have been\n",
      "dumped because their owners could not have borrowed money at any price to carry\n",
      "them. To prevent this was a considerable achievement for which all who owned\n",
      "stocks should have been thankful. But the banks received no credit. People\n",
      "remembered only that they had bravely undertaken to stem the price collapse and\n",
      "had failed.Despite a flattering supposition to the contrary, people come readily\n",
      "to terms with power. There is little reason to think that the power of the great\n",
      "bankers, while they were assumed to have it, was much resented. But as the\n",
      "ghosts of numerous tyrants, from Julius Caesar to Benito Mussolini will testify,\n",
      "people are very hard on those who, having had power, lose it or are destroyed.\n",
      "Then anger at past arrogance is joined with contempt for present weakness. The\n",
      "victim or his corpse is made to suffer all available indignities.Such was the\n",
      "fate of the bankers. For the next decade they were fair game for congressional\n",
      "committees, courts, the press, and the comedians. The great pretensions and the\n",
      "great failures of these days were a cause. A banker need not be popular; indeed,\n",
      "a good banker in a healthy capitalist society should probably be much disliked.\n",
      "People do not wish to trust their money to a hail-fellow-well-met but to a\n",
      "misanthrope who can say no. However, a banker must not seem futile, ineffective,\n",
      "or vaguely foolish. In contrast with the stern power of Morgan in 1907, that was\n",
      "precisely how his successors seemed, or were made to seem, in 1929.The failure\n",
      "of the bankers did not leave the community entirely without constructive\n",
      "leadership. There was Mayor James J. Walker. Appearing before a meeting of\n",
      "motion picture exhibitors on that Tuesday, he appealed to them to \"show pictures\n",
      "which will reinstate courage and hope in the hearts of the people.\"         VOn\n",
      "the Exchange itself, there was a strong feeling that corn-age and hope might\n",
      "best be restored by just closing up for a while. This feeling had, in fact, been\n",
      "gaining force for several days. Now it derived support from the simple\n",
      "circumstance that everyone was badly in need of sleep. Employees of some Stock\n",
      "Exchange firms had not been home for days. Hotel rooms in downtown New York were\n",
      "at a premium, and restaurants in the financial area had gone on to a fifteen-\n",
      "and twenty-hour day. Nerves were bad, and mistakes were becoming increasingly\n",
      "common. After the close of trading on Tuesday, a broker found a large waste\n",
      "basket of unexecuted orders which he had set aside for early attention and had\n",
      "totally forgotten.2 One customer, whose margin account was impaired, was sold\n",
      "out twice. A number of firms needed some time to see if they were still solvent.\n",
      "There were, in fact, no important failures by Stock Exchange firms during these\n",
      "days, although one firm had reported itself bankrupt as the result of a clerical\n",
      "error by an employee who was in the last stages of fatigue.3 Yet to close the\n",
      "Exchange was a serious matter. It might somehow signify that stocks had lost all\n",
      "their value, with consequences no one could foresee. In any case, securities\n",
      "would immediately become a badly frozen asset. This would be hard on the wholly\n",
      "solvent investors who might need to realize on them or use them as collateral.\n",
      "And sooner or later a new \"gutter\" market would develop in which individuals\n",
      "would informally dispose of stocks to those increasingly exceptional individuals\n",
      "who still wanted to buy them.In 1929 the New York Stock Exchange was in\n",
      "principle a sovereignty of its members. Apart from the general statutes relating\n",
      "to the conduct of business and the prevention of fraud, it was subject to no\n",
      "important state or federal regulation. This meant a considerable exercise of\n",
      "self-government. Legislation governing the conduct of trading had to be kept\n",
      "under review and enforced. Stocks had to be approved for listing. The building\n",
      "and other facilities of the Exchange had to be managed. As with the United\n",
      "States Congress, most of this work was done in committees. (These, in turn, were\n",
      "dominated by a somewhat smaller group of members who were expected and\n",
      "accustomed to run things.) A decision to close the Exchange had to be taken by\n",
      "the Governing Committee, a body of about forty members. The mere knowledge that\n",
      "this body was meeting would almost certainly have an unfavorable effect on the\n",
      "market. Nonetheless, at noon on Tuesday, the 29th, a meeting was held. The\n",
      "members of the committee left the floor in twos and threes and went, not to the\n",
      "regular meeting room, but to the office of the President of the Stock Clearing\n",
      "Corporation directly below the trading floor. Some months later, Acting\n",
      "President Whitney described the session with considerable graphic talent. \"The\n",
      "office they met in was never designed for large meetings of this sort, with the\n",
      "result that most of the Governors were compelled to stand, or to sit on tables.\n",
      "As the meeting proceeded, panic was raging overhead on the floor. Every few\n",
      "minutes the latest prices were announced, with quotations moving swiftly and\n",
      "irresistibly downwards. The feeling of those present was revealed by their habit\n",
      "of continually lighting cigarettes, taking a puff or two, putting them out and\n",
      "lighting new ones—a practice which soon made the narrow room blue with smoke and\n",
      "extremely stuffy.\"The result of these nervous deliberations was a decision to\n",
      "meet again in the evening. By evening the late rally had occurred, and it was\n",
      "decided to stay open for another day. The next day a further formula was hit\n",
      "upon. The Exchange would stay open. But it would have some special holidays and\n",
      "then go on short hours, and this would be announced just as soon as the market\n",
      "seemed strong enough to stand it.Many still wanted to close. Whitney said later,\n",
      "although no doubt with some exaggeration, that in the days to come \"the\n",
      "authorities of the Exchange led the life of hunted things, until [eventually]\n",
      "the desirability of holding the market open became apparent to all.\"\n",
      "VIThe next day those forces were at work which on occasion bring salvation\n",
      "precisely when salvation seems impossible. Stocks rose wonderfully,\n",
      "miraculously, though still on enormous volume. The Times industrials were up 31\n",
      "points for the day, thus recouping a large part of the terrible losses of the\n",
      "day before. Why this recovery occurred no one will ever know. Organized support\n",
      "can have no credit. Organized reassurance has a somewhat better claim. On the\n",
      "evening of the 29th, Dr. Julius Klein, Assistant Secretary of Commerce, friend\n",
      "of President Hoover, and the senior apostle of the official economic view, took\n",
      "to the radio to remind the country that President Hoover had said that the\n",
      "\"fundamental business of the country\" was sound. He added firmly, \"The main\n",
      "point which I want to emphasize is the fundamental soundness of [the] great mass\n",
      "of economic activities.\" On Wednesday, Waddill Catchings, of Goldman, Sachs,\n",
      "announced on return from a western trip that general business conditions were\n",
      "\"unquestionably fundamentally sound.\" (The same, by then, could not\n",
      "unquestionably be said of all Goldman, Sachs.) Arthur Brisbane told Hearst\n",
      "readers: \"To comfort yourself, if you lost, think of the people living near\n",
      "Mount Pelee, ordered to abandon their homes.\" Most important, perhaps, from\n",
      "Pocantico Hills came the first public statement by John D. Rockefeller in\n",
      "several decades. So far as the record shows, it was spontaneous. However,\n",
      "someone in Wall Street—perhaps someone who knew that another appeal to President\n",
      "Hoover to say something specifically encouraging about stocks would be\n",
      "useless—may have realized that a statement from Rockefeller would, if anything,\n",
      "be better. The statement ran: \"Believing that fundamental conditions of the\n",
      "country are sound ... my son and I have for some days been purchasing sound\n",
      "common stocks.\" The statement was widely applauded, although Eddie Cantor,\n",
      "describing himself as Comedian, Author, Statistician, and Victim, said later,\n",
      "\"Sure, who else had any money left?\" 4The accepted Wall Street explanation of\n",
      "Wednesday's miracle was not the reassurance but the dividend news of the day\n",
      "before. This also, without much question, was somewhat organized. U.S. Steel had\n",
      "declared an extra dividend; American Can had not only declared an extra but had\n",
      "increased its regular dividend. These errant sunbeams were deeply welcome in the\n",
      "dark canyons of lower Manhattan.Just before the Rockefeller statement arrived,\n",
      "things looked good enough on the Exchange so that Richard Whitney felt safe in\n",
      "announcing that the market would not open until noon the following day\n",
      "(Thursday) and that on Friday and Saturday it would stay shut. The announcement\n",
      "was greeted by cheers. Nerves were clearly past the breaking point. On La Salle\n",
      "Street in Chicago a boy exploded a firecracker. Like wildfire the rumor spread\n",
      "that gangsters whose margin accounts had been closed out were shooting up the\n",
      "street. Several squad cars of police arrived to make them take their losses like\n",
      "honest men. In New York the body of a commission merchant was fished out of the\n",
      "Hudson. The pockets contained $9.40 in change and some margin calls.\n",
      "VIIAt the short session of three hours on Thursday, October 31, well over seven\n",
      "million shares were traded, and the market made another good gain. The Times\n",
      "industrials were up 21 points. The weekly return of the Federal Reserve Bank\n",
      "showed a drop in brokers' loans by more than a billion, the largest weekly drop\n",
      "on record. Margin requirements had already been cut to 25 per cent; now the\n",
      "Federal Reserve Banks lowered the rediscount rate from 6 to 5 per cent. The\n",
      "Reserve Banks also launched vigorous open-market purchases of bonds to ease\n",
      "money rates and liberalize the supply of credit. The boom had collapsed; the\n",
      "restraint that had previously been contemplated could now give way to a policy\n",
      "of active encouragement to the market. On all these happy portents the market\n",
      "closed down for Friday, Saturday, and Sunday. They were not days of rest.\n",
      "Brokerage offices were fully staffed, and the Exchange floor was open for\n",
      "completion of trades and also for straightening out innumerable\n",
      "misunderstandings and mistakes. It was noted that on Friday a visitor to the\n",
      "galleries could not have told the market was suspended.The weekend brought one\n",
      "piece of bad news. That was the announcement on Saturday of the failure of the\n",
      "$20,000,000 Foshay enterprises of Minneapolis. Foshay owned utilities in some\n",
      "twelve states, Canada, Mexico, and Central America, and an assortment of hotels,\n",
      "flour mills, banks, manufacturing and retail establishments wherever he had\n",
      "happened to buy them. The 32-story obelisk, commemorating the enterprise, which\n",
      "still dominates the Minneapolis skyline, had been opened with fitting ceremony\n",
      "by Secretary of War James W. Good, only in August. (Secretary Good had referred\n",
      "to it as the \"Washington Monument of the Northwest.\")5 By all but the most\n",
      "technical of considerations, Foshay was bankrupt at that festive time. His\n",
      "survival depended on his ability to continue merchandising stock to the public.\n",
      "The market crash eliminated this source of revenue and made him dependent on the\n",
      "wholly inadequate earnings of his enterprises. On all other fronts the news was\n",
      "all good. Alfred P. Sloan, Jr., President of the General Motors Corporation,\n",
      "said: \"Business is sound.\" The Ford Motor Company emphasized a similar\n",
      "conviction by announcing a general reduction in its prices: \"...we feel that\n",
      "such a step is the best contribution that could be made to assure a continuation\n",
      "of good business.\" The Roadster was cut from $450 to $435; the Phaeton from $460\n",
      "to $440; the Tudor Sedan from $525 to $500. For the three days that the market\n",
      "was closed the papers carried stories of the accumulation of buying orders and,\n",
      "in some indefinable way, the stories had a greater ring of conviction than the\n",
      "week before. The market, after all, had closed after an excellent two-day rally.\n",
      "As Barron's pointed out, it could now be believed that stocks were selling \"ex-\n",
      "hopes and romance.\" On Monday, the Commercial National Bank and Trust Company\n",
      "took five columns in the Times to advertise \"...our belief and conviction that\n",
      "the general industrial and business condition of the country is fundamentally\n",
      "sound and is essentially unimpaired.\"That day the market started on another\n",
      "ghastly slump.         VIIIOver the weekend the financial community had almost\n",
      "certainly been persuaded by its own organized and spontaneous efforts at cheer.\n",
      "The papers described the reaction of professional Wall Street to Monday's market\n",
      "as one of stunned surprise, disbelief, and shock. Volume was smaller than the\n",
      "week before, but still well above six million. The whole list was weak;\n",
      "individual issues made big losses; the Times industrials were down 22 points for\n",
      "the day. Compared with anything but the week before, this was very bad. When\n",
      "measured against the bright hopes for that day, it was most\n",
      "distressing.Explanations varied. The rumor recurred that the \"organized support\"\n",
      "was selling stocks, and Mr. Lamont, on meeting the press, added a minor footnote\n",
      "to this now completed story. He said he didn't know—the organized support was\n",
      "really not that well organized. The most plausible explanation is that everyone\n",
      "was feeling cheerful but the public. As before and later, the weekend had been a\n",
      "time of thought, and out of thought had come pessimism and a decision to sell.\n",
      "So, as on other Mondays, no matter how cheerful the superficial portents, the\n",
      "selling orders poured in in volume.By now it was also evident that the\n",
      "investment trusts, once considered a buttress of the high plateau and a built-in\n",
      "defense against collapse, were really a profound source of weakness. The\n",
      "leverage, of which people only a fortnight before had spoken so knowledgeably\n",
      "and even affectionately, was now fully in reverse. With remarkable celerity it\n",
      "removed all of the value from the common stock of a trust. As before, the case\n",
      "of a typical trust, a small one, is worth contemplating. Let it be supposed that\n",
      "it had securities in the hands of the public which had a market value of\n",
      "$10,000,000 in early October. Of this, half was in common stock, half in bonds\n",
      "and preferred stock. These securities were fully covered by the current market\n",
      "value of the securities owned. In other words, the trust's portfolio contained\n",
      "securities with a market value also of $10,000,000. A representative portfolio\n",
      "of securities owned by such a trust would, in the early days of November, have\n",
      "declined in value by perhaps half. (Values of many of these securities by later\n",
      "standards would still be handsome; on November 4, the low for Tel and Tel was\n",
      "still 233, for General Electric it was 234, and for Steel 183.) The new\n",
      "portfolio value, $5,000,000, would be only enough to cover the prior claim on\n",
      "assets of the bonds and preferred stock. The common stock would have nothing\n",
      "behind it. Apart from expectations, which were by no means bright, it was now\n",
      "worthless.This geometrical ruthlessness was not exceptional. On the contrary, it\n",
      "was everywhere at work on the stock of the leverage trusts. By early November,\n",
      "the stock of most of them had become virtually unsalable. To make matters worse,\n",
      "many of them were traded on the Curb or the out-of-town exchanges where buyers\n",
      "were few and the markets thin.Never was there a time when more people wanted\n",
      "more money more urgently than in those days. The word that a man had \"got\n",
      "caught\" by the market was the signal for his creditors to descend on him like\n",
      "locusts. Many who were having trouble meeting their margin calls wanted to sell\n",
      "some stocks so they could hold the rest and thus salvage something from their\n",
      "misfortunes. But such people now found that their investment trust securities\n",
      "could not be sold for any appreciable sum and perhaps not at all. They were\n",
      "forced, as a result, to realize on their good securities. Standard stocks like\n",
      "Steel, General Motors, Tel and Tel were thus dumped on the market in abnormal\n",
      "volume, with the effect on prices that had already been fully revealed. The\n",
      "great investment trust boom had ended in a unique manifestation of Gresham's Law\n",
      "in which the bad stocks were driving out the good. The stabilizing effects of\n",
      "the huge cash resources of the investment trusts had also proved a mirage. In\n",
      "the early autumn the cash and liquid resources of the investment trusts were\n",
      "large. Many trusts had been attracted by the handsome returns in the call\n",
      "market. (The speculative circle had been closed. People who speculated in the\n",
      "stock of investment trusts were in effect investing in companies which provided\n",
      "the funds to finance their own speculation.) But now, as reverse leverage did\n",
      "its work, investment trust managements were much more concerned over the\n",
      "collapse in the value of their own stock than in the adverse movements in the\n",
      "stock list as a whole. The investment trusts had invested heavily in each other.\n",
      "As a result the fall in Blue Ridge hit Shenandoah, and the resulting collapse in\n",
      "Shenandoah was even more horrible for the Goldman Sachs Trading\n",
      "Corporation.Under these circumstances, many of the trusts used their available\n",
      "cash in a desperate effort to support their own stock. However, there was a vast\n",
      "difference between buying one's stock now when the public wanted to sell and\n",
      "buying during the previous spring—as Goldman Sachs Trading Corporation had\n",
      "done—when the public wanted to buy and the resulting competition had sent prices\n",
      "higher and higher. Now the cash went out and the stock came in, and prices were\n",
      "either not perceptibly affected or not for long. What six months before had been\n",
      "a brilliant financial maneuver was now a form of fiscal self-immolation. In the\n",
      "last analysis, the purchase by a firm of its own stock is the exact opposite of\n",
      "the sale of stocks. It is by the sale of stock that firms ordinarily\n",
      "grow.However, none of this was immediately apparent. If one has been a financial\n",
      "genius, faith in one's genius does not dissolve at once. To the battered but\n",
      "unbowed genius, support of the stock of one's own company still seemed a bold,\n",
      "imaginative, and effective course. Indeed, it seemed the only alternative to\n",
      "slow but certain death. So to the extent that their cash resources allowed, the\n",
      "managements of the trusts chose faster, though equally certain death. They\n",
      "bought their own worthless stock. Men have been swindled by other men on many\n",
      "occasions. The autumn of 1929 was, perhaps, the first occasion when men\n",
      "succeeded on a large scale in swindling themselves.The time has now come to\n",
      "complete the chronicle of the last days of the crisis.         IXTuesday,\n",
      "November 5, was election day, and the market was closed all day. In the New York\n",
      "mayoralty race, the Democratic incumbent, James J. Walker, scored a landslide\n",
      "victory over his Republican opponent, F. H. La Guardia, who had been soundly\n",
      "denounced by the Democrats as a socialist. Babson, in a statement, called for\n",
      "poise, discernment, judicious coin-age, and old-fashioned common sense. On\n",
      "Wednesday the market reopened for the first of a new series of short sessions of\n",
      "three hours. These were the compromise on the question of closing which had been\n",
      "reached the previous week. Nearly six million shares were traded in this session\n",
      "or the equivalent of ten million shares on a full day. There was another\n",
      "sickening slide. U.S. Steel opened at 181, and, by what one paper called a\n",
      "succession of \"feverish dips,\" went to 165. Auburn Automobile lost 66 points;\n",
      "Otis Elevator lost 45. The Times industrials were off 37 points for the day, or\n",
      "only 6 points less than on the terrible Tuesday eight days earlier. Where would\n",
      "it all end? There was also disturbing news from beyond the market. Fundamentals\n",
      "seemed to be turning sour. The week's figures on carloadings showed a heavy drop\n",
      "as compared with the year before. The steel rate was significantly down from the\n",
      "preceding week. More serious, the slump had extended to the commodity markets.\n",
      "On previous days these had reacted sympathetically with the stock market. On\n",
      "this Wednesday they had troubles of their own. Cotton was sharply off in the\n",
      "heaviest trading in weeks. References were made to \"panic\" in the wheat market\n",
      "when the price dropped vertically at noon.On Thursday the stock market was\n",
      "steady to higher, but on Friday it took a small drop. People had another weekend\n",
      "of contemplation. This time there was no talk of an accumulation of buying\n",
      "orders; indeed, there was little good news of any kind. On Monday, November 11,\n",
      "came another drastic slump. For the next two days trading was heavy—the Exchange\n",
      "was still on short hours—and prices went down still more. In these three days,\n",
      "November 11, 12, and 13, the Times industrials lost another 50 points.Of all the\n",
      "days of the crash, these without doubt were the dreariest. Organized support had\n",
      "failed. For the moment even organized reassurance had been abandoned. All that\n",
      "could be managed was some sardonic humor. It was noted that the margin calls\n",
      "going out by Western Union that week carried a small sticker: \"Remember them at\n",
      "home with a cheery Thanksgiving telegram, the American way for this American\n",
      "day.\" Clerks in downtown hotels were said to be asking guests whether they\n",
      "wished the room for sleeping or jumping. Two men jumped hand-in-hand from a high\n",
      "window in the Ritz. They had a joint account. The Wall Street Journal, becoming\n",
      "biblical, told its readers: \"Verily, I say, let the fear of the market be the\n",
      "law of thy life, and abide by the words of the bond salesman.\" The financial\n",
      "editor of the Times, who by this time showed signs of being satisfied with the\n",
      "crash and perhaps even of feeling that it had gone too far, said: \"Probably none\n",
      "of the present generation will be able to speak again ... of a 'healthy\n",
      "reaction.' There are many signs that the phrase is entirely out of date.\"\n",
      "CHAPTER VIIAftermath IIN THE WEEK or so following Black Thursday, the London\n",
      "penny press told delightedly of the scenes in downtown New York. Speculators\n",
      "were hurling themselves from windows; pedestrians picked their way delicately\n",
      "between the bodies of fallen financiers. The American correspondent for The\n",
      "Economist wrote an indignant column for his paper protesting against this\n",
      "picture of imaginary carnage.In the United States the suicide wave that followed\n",
      "the stock market crash is also a part of the legend of 1929. In fact, there was\n",
      "none. For several years before 1929, the suicide rate had been gradually rising.\n",
      "It continued to increase in that year, with a further and much sharper increase\n",
      "in 1930, 1931, and 1932—years when there were many things besides the stock\n",
      "market to cause people to conclude that life was no longer worth living. The\n",
      "statistics for New Yorkers, who might be assumed to have had a special\n",
      "propensity for self-destruction, derived from their special propinquity to the\n",
      "market, show only a slight deviation from those for the country as a whole.\n",
      "Since the suicide myth is so well established, it may be useful to give the\n",
      "detailed figures. They are as follows:  NUMBER OF SUICIDES PER 100,000 OF\n",
      "POPULATION 1925–34 Year For Registration Area* For New York City 1925 12.1 14.4\n",
      "1926 12.8 13.7 1927 13.3 15.7 1928 13.6 15.7 1929 14.0 17.0 1930 15.7 18.7 1931\n",
      "16.8 19.7 1932 17.4 21.3 1933 15.9 18.5 1934 14.9 17.0  Since the market crash\n",
      "took place late in the year, there could have been a substantial increase in\n",
      "suicides in late October and thereafter which still would not be great enough to\n",
      "affect the figures for the year as a whole. However, figures on the causes of\n",
      "death by months are also available for 1929.1 These show that the number of\n",
      "suicides in October and November was comparatively low—in October, 1331 suicides\n",
      "in all the United States, and in November, 1344. In only three other\n",
      "months—January, February, and September—did fewer people destroy themselves.\n",
      "During the summer months, when the market was doing beautifully, the number of\n",
      "suicides was substantially higher. One can only guess how the suicide myth\n",
      "became established. Like alcoholics and gamblers, broken speculators are\n",
      "supposed to have a propensity for self-destruction. At a time when broken\n",
      "speculators were plentiful, the newspapers and the public may have simply\n",
      "supplied the corollary. Alternatively, suicides that in other times would have\n",
      "evoked the question, \"Why do you suppose he did it?\" now had the motive assigned\n",
      "automatically: \"The poor fellow was caught in the crash.\" Finally, it must be\n",
      "noted that, although suicides did not increase sharply either in the months of\n",
      "the crash or in 1929 as a whole, the rate did rise in the later depression\n",
      "years. In memory some of these tragedies may have been moved back a year or two\n",
      "to the time of the stock market crash.The weight of the evidence suggests that\n",
      "the newspapers and the public merely seized on such suicides as occurred to show\n",
      "that people were reacting appropriately to their misfortune. Enough deaths could\n",
      "be related in one way or another to the market to serve. Beginning soon after\n",
      "Black Thursday, stories of violent self-destruction began to appear in the\n",
      "papers with some regularity. Curiously, though another myth runs strongly to the\n",
      "contrary, few people in these days followed the classical method of jumping from\n",
      "a high window. One would-be suicide jumped into the Schuylkill River, although\n",
      "he changed his mind when he hit the water and was fished out. The head of the\n",
      "Rochester Gas and Electric Company took gas. Another martyr dipped himself in\n",
      "gasoline and touched himself off. He not only made good his escape from his\n",
      "margin calls, but took his wife with him. There was also the suicide of J. J.\n",
      "Riordan.Riordan's death made large headlines in the newspapers on Sunday,\n",
      "November 10. The papers obviously had sensed a story not only in his death\n",
      "itself, but also in the manner of its announcement. Riordan was a widely-known\n",
      "and popular figure among New York Democrats. He had been treasurer of one of\n",
      "Mayor Walker's campaigns and also of one of Al Smith's. He and Smith were close\n",
      "friends and business associates. Al Smith was a member of the board of directors\n",
      "of the recently-organized County Trust Company, of which Riordan was the\n",
      "president. On Friday, November 8, Riordan went to his bank, took a pistol from a\n",
      "teller's cage, went home and shot himself. Al Smith was notified, and his sorrow\n",
      "over the death of his friend was not diminished by the knowledge that the news\n",
      "might start a serious run on their bank. A medical examiner was called but\n",
      "further notification was withheld until the following day (Saturday) at noon,\n",
      "when the bank had closed for the weekend. There had been a long wake through\n",
      "which the distinguished mourners had kept one eye on the corpse and the other on\n",
      "the clock.The medical examiner first implied that he had postponed notification\n",
      "out of a feeling of deep responsibility for depositors of the County Trust. This\n",
      "was a formidable exercise of discretion; carried to its logical conclusion, it\n",
      "meant that all deaths would have to be weighed by the attending doctor for their\n",
      "financial consequences. Later it was tacitly conceded that the decision was Al\n",
      "Smith's. So great was Smith's prestige—and also the general nervousness—that the\n",
      "action was not questioned.For some days rumors had been circulating that Riordan\n",
      "had been wiped out by the crash. Now his friends rallied to his defense, some\n",
      "with vehement assertions that he never played the market. He had been deeply\n",
      "involved, as subsequent Senate committee investigations of the stock market\n",
      "revealed, but a hurried audit of the bank showed that all of its funds were\n",
      "intact. This fact was well-publicized over the weekend. The City Administration\n",
      "boldly announced that it was leaving its deposits with the bank, which was a\n",
      "trifle like saying that it would remain on speaking terms with Tammany Hall.\n",
      "Raskob temporarily assumed the chairmanship. No run developed. The Church\n",
      "concluded that Riordan, a Catholic, had been temporarily deranged and thus was\n",
      "eligible for burial in consecrated ground. Among the honorary pallbearers were\n",
      "Al Smith, Herbert Lehman, and John J. Raskob, and among those attending the\n",
      "funeral were Mayor Frank Hague, Vincent Astor, Grover Whalen, James A. Farley,\n",
      "and M. J. Meehan, the market operator.Two and a half years later, on Saturday,\n",
      "March 12, 1932, Ivar Kreuger shot himself dead in his Paris apartment at eleven\n",
      "o'clock in the morning local time. This was six hours before the New York Stock\n",
      "Exchange closed. With the cooperation of the Paris police, the news was withheld\n",
      "until the market closed. Later a congressional committee was exceedingly\n",
      "critical of this delay, and Al Smith's action was cited in the defense. In the\n",
      "case of Kreuger, it should be added, the security system of the Paris police was\n",
      "less than perfect. It is fairly certain that there was heavy selling that\n",
      "morning—including heavy short selling—of Kreuger and Toll by continental\n",
      "interests.2         IIIn many ways the effect of the crash on embezzlement was\n",
      "more significant than on suicide. To the economist embezzlement is the most\n",
      "interesting of crimes. Alone among the various forms of larceny it has a time\n",
      "parameter. Weeks, months, or years may elapse between the commission of the\n",
      "crime and its discovery. (This is a period, incidentally, when the embezzler has\n",
      "his gain and the man who has been embezzled, oddly enough, feels no loss. There\n",
      "is a net increase in psychic wealth.) At any given time there exists an\n",
      "inventory of undiscovered embezzlement in—or more precisely not in—the country's\n",
      "businesses and banks. This inventory—it should perhaps be called the\n",
      "bezzle—amounts at any moment to many millions of dollars. It also varies in size\n",
      "with the business cycle. In good times people are relaxed, trusting, and money\n",
      "is plentiful. But even though money is plentiful, there are always many people\n",
      "who need more. Under these circumstances the rate of embezzlement grows, the\n",
      "rate of discovery falls off, and the bezzle increases rapidly. In depression all\n",
      "this is reversed. Money is watched with a narrow, suspicious eye. The man who\n",
      "handles it is assumed to be dishonest until he proves himself otherwise. Audits\n",
      "are penetrating and meticulous. Commercial morality is enormously improved. The\n",
      "bezzle shrinks. The stock market boom and the ensuing crash caused a traumatic\n",
      "exaggeration of these normal relationships. To the normal needs for money, for\n",
      "home, family and dissipation, was added, during the boom, the new and\n",
      "overwhelming requirement for funds to play the market or to meet margin calls.\n",
      "Money was exceptionally plentiful. People were also exceptionally trusting. A\n",
      "bank president who was himself trusting Kreuger, Hopson, and Insull was\n",
      "obviously unlikely to suspect his lifelong friend the cashier. In the late\n",
      "twenties the bezzle grew apace.Just as the boom accelerated the rate of growth,\n",
      "so the crash enormously advanced the rate of discovery. Within a few days,\n",
      "something close to universal trust turned into something akin to universal\n",
      "suspicion. Audits were ordered. Strained or preoccupied behavior was noticed.\n",
      "Most important, the collapse in stock values made irredeemable the position of\n",
      "the employee who had embezzled to play the market. He now confessed. After the\n",
      "first week or so of the crash, reports of defaulting employees were a daily\n",
      "occurrence. They were far more common than the suicides. On some days\n",
      "comparatively brief accounts occupied a column or more in the Times. The amounts\n",
      "were large and small, and they were reported from far and wide.The most\n",
      "spectacular embezzlement of the period—the counterpart of the Riordan\n",
      "suicide—was the looting of the Union Industrial Bank of Flint, Michigan. The\n",
      "gross take, estimates of which grew alarmingly as the investigation proceeded,\n",
      "was stated in The Literary Digest later in the year to be $3,592,000.3In the\n",
      "beginning this embezzlement was a matter of individual initiative. Unknown to\n",
      "each other, a number of the bank's officers began making away with funds.\n",
      "Gradually they became aware of each other's activities, and since they could\n",
      "scarcely expose each other, they co-operated. The enterprise eventually embraced\n",
      "about a dozen people, including virtually all of the principal officers of the\n",
      "bank. Operations were so well organized that even the arrival of bank examiners\n",
      "at the local hotels was made known promptly to members of the syndicate.Most of\n",
      "the funds which were purloined had been deposited with the bank to be loaned in\n",
      "the New York call market. The money was duly dispatched to New York but promptly\n",
      "recalled while the records continued to show that it was there. The money was\n",
      "then returned once more to New York and put into stocks. In the spring of 1929\n",
      "the group was about $100,000 ahead. Then, unfortunately, it went short just as\n",
      "the market soared into the blue yonder of the summer sky. This was so costly\n",
      "that the group was induced to return to a long position, which it did just\n",
      "before the crash. The crash, needless to say, was mortal.Each week during the\n",
      "autumn more such unfortunates were revealed in their misery. Most of them were\n",
      "small men who had taken a flier in the market and then become more deeply\n",
      "involved. Later they had more impressive companions. It was the crash, and the\n",
      "subsequent ruthless contraction of values which, in the end, exposed the\n",
      "speculation by Kreuger, Hopson, and Insull with the money of other people.\n",
      "Should the American economy ever achieve permanent full employment and\n",
      "prosperity, firms should look well to their auditors. One of the uses of\n",
      "depression is the exposure of what auditors fail to find. Bagehot once observed:\n",
      "\"Every great crisis reveals the excessive speculations of many houses which no\n",
      "one before suspected.\"4         IIIIn mid-November 1929, at long, long last, the\n",
      "market stopped falling—at least, for a while. The low was on Wednesday, November\n",
      "13. On that day the Times industrials closed at 224 down from 452, or by almost\n",
      "exactly one half since September 3. They were also by then down 82 points—about\n",
      "one quarter—from the close on that day barely two weeks before when John D.\n",
      "Rockefeller had announced that he and his son were buying common stocks. On\n",
      "November 13 there was another Rockefeller story: it was said that the family had\n",
      "entered a million-share buying order to peg Standard Oil of New Jersey at 50.\n",
      "During the rest of November and December the course of the market was moderately\n",
      "up. The decline had run its course. However, the end coincided with one last\n",
      "effort at reassurance. No one can say for sure that it did no good. One part was\n",
      "the announcement by the New York Stock Exchange of an investigation of short\n",
      "selling. Inevitably in the preceding weeks there had been rumors of bear raids\n",
      "on the market and of fortunes being made by the shorts. The benign people known\n",
      "as \"they,\" who once had put the market up, were now a malign influence putting\n",
      "it down and making money out of the common disaster. In the early days of the\n",
      "crash it was widely believed that Jesse L. Livermore, a Bostonian with a large\n",
      "and unquestionably exaggerated reputation for bear operations, was heading a\n",
      "syndicate that was driving the market down. So persistent did these rumors\n",
      "become that Livermore, whom few had thought sensitive to public opinion, issued\n",
      "a formal denial that he was involved in any deflationary plot. \"What little\n",
      "business I have done in the stock market,\" he said, \"has always been as an\n",
      "individual and will continue to be done on such basis.\" As early as October 24,\n",
      "The Wall Street Journal, then somewhat less reserved in its view of the world\n",
      "than now, complained that \"there has been a lot of short selling, a lot of\n",
      "forced selling, and a lot of selling to make the market look bad.\" Such\n",
      "suspicions the Exchange authorities now sought to dispel. Nothing came of the\n",
      "study.A more important effort at reassurance was made by President Hoover.\n",
      "Presumably he was still indifferent to the fate of the stock market. But he\n",
      "could not be indifferent to the much publicized fundamentals, which by now were\n",
      "behaving worse each week. Prices of commodities were falling. Freight-car\n",
      "loadings, pig iron and steel production, coal output, and automobile production\n",
      "were also all going down. So, as a result, was the general index of industrial\n",
      "production. Indeed, it was falling much more rapidly than in the sharp postwar\n",
      "depression of 1920–21. There were alarming stories of the drop in consumer\n",
      "buying, especially of more expensive goods. It was said that sales of radio sets\n",
      "in New York had fallen by half since the crash. Mr. Hoover's first step was out\n",
      "of the later works of John Maynard Keynes. Precisely as Keynes and Keynesians\n",
      "would have advised, he announced a cut in taxes. The rate on both individuals\n",
      "and corporations was cut by one full percentage point. This reduced the income\n",
      "tax of a head of a family with no dependents and an income of $4000 by two-\n",
      "thirds. The man with $5000 got a similar reduction. The tax of a married man\n",
      "with no dependents and an income of $10,000 was cut in half. These were dramatic\n",
      "reductions, but their effect was sadly mitigated by the fact that for most\n",
      "people the taxes being cut were already insignificant. The man with $4000 had\n",
      "his annual tax burden reduced from $5.63 to $1.88. The man with $5000 got a cut\n",
      "from $16.88 to $5.63. For the man with $10,000 the reduction in annual tax was\n",
      "from $120 to $65. The step, nonetheless, was well received as a contribution to\n",
      "increased purchasing power, expanded business investment, and a general revival\n",
      "of confidence.Mr. Hoover also called a series of meetings on the state of the\n",
      "economy. The leading industrialists, the leading railway executives, the heads\n",
      "of the large utilities, the heads of the important construction companies, the\n",
      "union leaders, and the heads of the farm organizations met in turn with the\n",
      "President during the latter part of November. The procedure in the case of each\n",
      "of the meetings was the same. There was a solemn session with the President,\n",
      "those attending had their picture taken with the President, and there was a\n",
      "press interview at which the conferees gave the press their opinion on the\n",
      "business prospect. The latter, without exception, was highly favorable. After\n",
      "the meeting of the industrial leaders on November 21, which was attended by,\n",
      "among others, Henry Ford, Walter Teagle, Owen D. Young, Alfred P. Sloan, Jr.,\n",
      "Pierre du Pont, Walter Gifford, and Andrew Mellon, the expressions of confidence\n",
      "were so robust that Julius Rosenwald, who also attended, said he feared there\n",
      "might soon be a bad labor shortage.The utility, rail, and construction\n",
      "executives were equally hopeful. Even the heads of the farm organizations were\n",
      "less misanthropic than normal for that time. They said afterward that they had\n",
      "told the President that \"the morale of their industry was better than it had\n",
      "been for years.\" 5This was organized reassurance on a really grand scale, and it\n",
      "attracted some of the most enthusiastic comment of the period. A Wall Street\n",
      "financial writer began his story of the sessions: \"'Order up the Moors!' was\n",
      "Marshal Foch's reply at the first battle of the Marne... 'Order up the business\n",
      "reserves,' directed President Hoover as pessimistic reports flowed in from all\n",
      "quarters following the stock market crash.\" The Philadelphia Record was led to\n",
      "describe the President as \"easily the most commanding figure in the modern\n",
      "science of engineering statesmanship.'\" The Boston Globe said that the nation is\n",
      "now aware \"that it has at the White House a man who believes not in the\n",
      "philosophy of drift, but in the dynamics of mastery.\" 6         IVYet to suppose\n",
      "that President Hoover was engaged only in organizing further reassurance is to\n",
      "do him a serious injustice. He was also conducting one of the oldest, most\n",
      "important—and, unhappily, one of the least understood—rites in American life.\n",
      "This is the rite of the meeting which is called not to do business but to do no\n",
      "business. It is a rite which is still much practiced in our time. It is worth\n",
      "examining for a moment. Men meet together for many reasons in the course of\n",
      "business. They need to instruct or persuade each other. They must agree on a\n",
      "course of action. They find thinking in public more productive or less painful\n",
      "than thinking in private. But there are at least as many reasons for meetings to\n",
      "transact no business. Meetings are held because men seek companionship or, at a\n",
      "minimum, wish to escape the tedium of solitary duties. They yearn for the\n",
      "prestige which accrues to the man who presides over meetings, and this leads\n",
      "them to convoke assemblages over which they can preside. Finally, there is the\n",
      "meeting which is called not because there is business to be done, but because it\n",
      "is necessary to create the impression that business is being done. Such meetings\n",
      "are more than a substitute for action. They are widely regarded as action.The\n",
      "fact that no business is transacted at a no-business meeting is normally not a\n",
      "serious cause of embarrassment to those attending. Numerous formulas have been\n",
      "devised to prevent discomfort. Thus scholars, who are great devotees of the no-\n",
      "business meeting, rely heavily on the exchange-of-ideas justification. To them\n",
      "the exchange of ideas is an absolute good. Any meeting at which ideas are\n",
      "exchanged is, therefore, useful. This justification is nearly ironclad. It is\n",
      "very hard to have a meeting of which it can be said that no ideas were\n",
      "exchanged.Salesmen and sales executives, who also are important practitioners of\n",
      "the no-business gathering, commonly have a different justification and one that\n",
      "has strong spiritual overtones. Out of the warmth of comradeship, the interplay\n",
      "of personality, the stimulation of alcohol, and the inspiration of oratory comes\n",
      "an impulsive rededication to the daily task. The meeting pays for itself in a\n",
      "fuller and better life and the sale of more goods in future weeks and months.\n",
      "The no-business meetings of the great business executives depend for their\n",
      "illusion of importance on something quite different. Not the exchange of ideas\n",
      "or the spiritual rewards of comradeship, but a solemn sense of assembled power\n",
      "gives significance to this assemblage. Even though nothing of importance is said\n",
      "or done, men of importance cannot meet without the occasion seeming important.\n",
      "Even the commonplace observation of the head of a large corporation is still the\n",
      "statement of the head of a large corporation. What it lacks in content it gains\n",
      "in power from the assets back of it.The no-business meeting was an almost\n",
      "perfect instrument for the situation in which President Hoover found himself in\n",
      "the autumn of 1929. The modest tax cut apart, the President was clearly averse\n",
      "to any large-scale government action to counter the developing depression. Nor\n",
      "was it very certain, at the time, what could be done. Yet by 1929 popular faith\n",
      "in laissez faire had been greatly weakened. No responsible political leader\n",
      "could safely proclaim a policy of keeping hands off. The no-business meetings at\n",
      "the White House were a practical expression of laissez faire. No positive action\n",
      "resulted. At the same time they gave a sense of truly impressive action. The\n",
      "conventions governing the no-business session insured that there would be no\n",
      "embarrassment arising from the absence of business. Those who attended accepted\n",
      "as a measure of the importance of the meetings the importance of the people\n",
      "attending. The newspapers also cooperated in emphasizing the importance of the\n",
      "sessions. Had they done otherwise they would, of course, have undermined the\n",
      "value of the sessions as news.In recent times the no-business meeting at the\n",
      "White House—attended by governors, industrialists, representatives of business,\n",
      "labor, and agriculture—has become an established institution of government. Some\n",
      "device for simulating action, when action is impossible, is indispensable in a\n",
      "sound and functioning democracy. Mr. Hoover in 1929 was a pioneer in this field\n",
      "of public administration.As the depression deepened, it was said that Mr.\n",
      "Hoover's meetings had been a failure. This, obviously, reflects a very narrow\n",
      "view.         VIn January, February, and March of 1930 the stock market showed a\n",
      "substantial recovery. Then in April the recovery lost momentum, and in June\n",
      "there was another large drop. Thereafter, with few exceptions the market dropped\n",
      "week by week, month by month, and year by year through June of 1932. The\n",
      "position when it finally halted made the worst level during the crash seem\n",
      "memorable by contrast. On November 13, 1929, it may be recalled, the Times\n",
      "industrials closed at 224. On July 8, 1932, they were 58. This value was not\n",
      "much more than the net by which they dropped on the single day of October 28,\n",
      "1929. Standard Oil of New Jersey, which the Rockefellers were believed to have\n",
      "pegged at 50 on November 13, 1929, dropped below 20 in April 1932. On July 8 it\n",
      "was 24. U.S. Steel on July 8 reached a low of 22. On September 3, 1929, it had\n",
      "sold as high as 262. General Motors was a bargain at 8 on July 8, down from 73\n",
      "on September 3, 1929. Montgomery Ward was 4, down from 138. Tel and Tel was 72,\n",
      "and on September 3, 1929, it had sold at 304. Anaconda sold at 4 on July 8. The\n",
      "Commercial and Financial Chronicle observed that \"the copper shares are so low\n",
      "that their fluctuations are of little consequence.\"7 However, comparatively\n",
      "speaking, values in these staple stocks had been well maintained. Things were\n",
      "far worse with the investment trusts. Blue Ridge during the week ending July 8,\n",
      "1932, was 63 cents, and Shenandoah was 50 cents. United Founders and American\n",
      "Founders were both around 50 cents as compared with 70 and 117 (dollars,\n",
      "needless to say) on September 3, 1929. The fears of November 1929 that the\n",
      "investment trusts might go to nothing had been largely realized.No one any\n",
      "longer suggested that business was sound, fundamentally or otherwise. During the\n",
      "week of July 8, 1932, Iron Age announced that steel operations had reached 12\n",
      "per cent of capacity. This was thought of its sort to be a record. Pig iron\n",
      "output was the lowest since 1896. A total of 720,278 shares were traded that day\n",
      "on the New York Stock Exchange.Before all this came to pass there had been more,\n",
      "many more, efforts at reassurance. In the weeks of the crash President Hoover\n",
      "had sagely observed: \"My own experience ... has been that words are not of any\n",
      "great importance in times of economic disturbance.\" This impregnable rule he\n",
      "thereafter forgot. In December he told the Congress that the steps he had\n",
      "taken—the White House no-business conferences in particular—had \"re-established\n",
      "confidence.\" In March 1930, following a flood of optimistic forecasts by his\n",
      "subordinates, Mr. Hoover said that the worst effect of the crash upon\n",
      "unemployment would be ended in sixty days. In May Mr. Hoover said he was\n",
      "convinced \"we have now passed the worst and with continued unity of effort shall\n",
      "rapidly recover.\" Toward the end of the month he said that business would be\n",
      "normal by fall.8 What was perhaps the last word on the policy of reassurance was\n",
      "said by Simeon D. Fess, the Chairman of the Republican National\n",
      "Committee: Persons high in Republican circles are beginning to believe that\n",
      "there is some concerted effort on foot to utilize the stock market as a method\n",
      "of discrediting the Administration. Every time an Administration official gives\n",
      "out an optimistic statement about business conditions, the market immediately\n",
      "drops.9          CHAPTER VIIIAftermath IITHE CRASH blighted the fortunes of many\n",
      "hundreds of thousands of Americans. But among people of prominence worse havoc\n",
      "was worked on reputations. In such circles credit for wisdom, foresight, and,\n",
      "unhappily also, for common honesty underwent a convulsive shrinkage.On the\n",
      "whole, those who had proclaimed during the crash that business was\n",
      "\"fundamentally sound\" were not held accountable for their words. The ritualistic\n",
      "nature of their expression was recognized; then as now no one supposed that such\n",
      "spokesmen knew whether business was sound or unsound. One exception was Mr.\n",
      "Hoover. He undoubtedly suffered as the result of his repeated predictions of\n",
      "imminent prosperity. However, Hoover had converted the simple business ritual of\n",
      "reassurance into a major instrument of public policy. It was certain in\n",
      "consequence to be the subject of political comment.The scholarly forecasters\n",
      "were not so fortunate. People on the whole cherished the discovery that they\n",
      "were not omniscient. Mr. Lawrence disappeared from Princeton. Among economists\n",
      "his voice was not heard again.The Harvard Economic Society, it will be recalled,\n",
      "had come up to the summer of the crash with a valuable reputation for pessimism.\n",
      "This position it abandoned during the summer when the stock market kept on\n",
      "rising and business seemed strong. On November 2, after the crash, the Society\n",
      "concluded that \"the present recession, both for stocks and business, is not the\n",
      "precursor of business depression.\" On November 10 it made its notable estimate\n",
      "that \"a serious depression like that of 1920–21 is outside the range of\n",
      "probability.\" It repeated this judgment on November 23 and on December 21 gave\n",
      "its forecast for the new year: \"A depression seems improbable; [we expect]\n",
      "recovery of business next spring, with further improvement in the fall.\" On\n",
      "January 18, 1930, the Society said, \"There are indications that the severest\n",
      "phase of the recession is over\"; on March 1, that \"manufacturing activity is\n",
      "now—to judge from past periods of contraction—definitely on the road to\n",
      "recovery\", on March 22, \"The outlook continues favorable\"; on March 29, that\n",
      "\"the outlook is favorable\"; on April 19, that \"by May or June the spring\n",
      "recovery forecast in our letters of last December and November should be clearly\n",
      "apparent\"; on May 17, that business \"will turn for the better this month or\n",
      "next, recover vigorously in the third quarter and end the year at levels\n",
      "substantially above normal\"; on May 24 it was suggested that conditions\n",
      "\"continue to justify\" the forecasts of May 17; on June 21, that \"despite\n",
      "existing irregularities\" there would soon be improvement; on June 28 it stated\n",
      "that \"irregular and conflicting movements of business should soon give way to\n",
      "sustained recovery\"; on July 19 it pointed out that \"untoward elements have\n",
      "operated to delay recovery but the evidence nonetheless points to substantial\n",
      "improvement\"; and on August 30, 1930, the Society stated that \"the present\n",
      "depression has about spent its force.\" Thereafter the Society became less\n",
      "hopeful. On November 15, 1930, it said: \"We are now near the end of the\n",
      "declining phase of the depression.\" A year later, on October 31, 1931, it said:\n",
      "\"Stabilization at [present] depression levels is clearly possible.\" 1 Even these\n",
      "last forecasts were wildly optimistic. Somewhat later, its reputation for\n",
      "infallibility rather dimmed, the Society was dissolved. Harvard economics\n",
      "professors ceased forecasting the future and again donned their accustomed garb\n",
      "of humility. Professor Irving Fisher tried hard to explain why he had been\n",
      "wrong. Early in November 1929 he suggested that the whole thing had been\n",
      "irrational and hence beyond prediction. In a statement that was not a model of\n",
      "coherence, he said: \"It was the psychology of panic. It was mob psychology, and\n",
      "it was not, primarily, that the price level of the market was unsoundly high ...\n",
      "the fall in the market was very largely due to the psychology by which it went\n",
      "down because it went down.\"2 The explanation attracted little attention except\n",
      "from the editor of The Commercial and Financial Chronicle. The latter observed\n",
      "with succinct brutality: \"The learned professor is wrong as he usually is when\n",
      "he talks about the stock market.\" The \"mob,\" he added, didn't sell. It got sold\n",
      "out.Before the year was over, Professor Fisher tried again in a book, The Stock\n",
      "Market Crash—and After.3 He argued, and rightly for the moment, that stocks were\n",
      "still on a plateau, albeit a somewhat lower one than before, that the crash was\n",
      "a great accident, that the market had gone up \"principally because of sound,\n",
      "justified expectations of earnings.\" He also argued that prohibition was still a\n",
      "strong force for higher business productivity and profits, and concluded that\n",
      "for \"the immediate future, at least, the outlook is bright.\" This book attracted\n",
      "little attention. One trouble with being wrong is that it robs the prophet of\n",
      "his audience when he most needs it to explain why.Out in Ohio, Professor Dice—he\n",
      "of the parasangs—survived honorably to write and teach for a lifetime about\n",
      "finance.This may be the place also to record another happy ending. Goldman,\n",
      "Sachs and Company rescued its firm name from its delinquent offspring and\n",
      "returned to an earlier role of strict rectitude and stern conservatism. It\n",
      "became known for its business in the most austere of securities.         IINew\n",
      "York's two greatest banks, the Chase and the National City, suffered severely in\n",
      "the aftermath. They, of course, shared the general obloquy of the New York\n",
      "bankers which resulted from the great hopes and great disappointments of\n",
      "organized support. But it was also the remarkable misfortune of each that it had\n",
      "as its head in those days a market operator in the grand manner.Of the two, the\n",
      "Chase was the more fortunate. Albert H. Wiggin, variously President, Chairman of\n",
      "the board, and Chairman of the governing board of the Chase, was a speculator\n",
      "and operator, but not an articulate one. However, in 1929 and the years\n",
      "preceding he had been engaging in some astonishing enterprises. In 1929 he\n",
      "received $275,000 compensation as head of the Chase. He was also—or while head\n",
      "of the Chase had been—director of some fifty-nine utility, industrial,\n",
      "insurance, and other corporations and from some of these had also received a\n",
      "handsome salary. Armour and Company had paid him $40,000 to be a member of its\n",
      "finance committee; he got $20,000 a year from the Brooklyn-Manhattan Transit\n",
      "Corporation; at least seven other firms paid him from two to five thousand\n",
      "annually.4 Wisdom and esteem, or even affection, were not the only factors in\n",
      "this compensation. Those who paid were usually clients and prospective borrowers\n",
      "from the Chase. But the most remarkable of Mr. Wiggin's extracurricular\n",
      "interests was a bevy of private companies. Three were personal holding\n",
      "companies, two of which were named sentimentally for his daughters. Three others\n",
      "were incorporated in Canada for highly unsentimental reasons of taxation and\n",
      "corporate reticence.5 These companies were the instruments for an astonishing\n",
      "variety of stock market operations. In one operation in the spring of 1929\n",
      "Shermar Corporation—one of the namesake companies—participated with Harry F.\n",
      "Sinclair and Arthur W. Cutten in a mammoth pool in the common stock of Sinclair\n",
      "Consolidated Oil Company. Even in those tolerant days Sinclair and Cutten were\n",
      "considered rather garish companions for a prominent banker. However, the\n",
      "operation netted Shermar $891,600.37 on no apparent investment.6However, the\n",
      "most breathtaking of Mr. Wiggin's operations were in the stock of the Chase\n",
      "National Bank. These, in turn, were financed by the Chase bank itself. In one\n",
      "exceptionally well-timed coup Shermar Corporation, between September 23 and\n",
      "November 4, 1929, sold short 42,506 shares of Chase stock. (For those to whom\n",
      "short selling is an unrevealed mystery, this meant in effect that it negotiated\n",
      "a loan of 42,506 shares and then sold them at the exceedingly good prices then\n",
      "obtaining. It did this with the intention of buying the same number of shares\n",
      "later at a lower price in order to repay in kind the lender who had provided the\n",
      "original stock. The profit from repaying shares bought at a lower price—always\n",
      "assuming that the price went down—would obviously accrue to Shermar.) Prices did\n",
      "go down superbly; the short sale anticipated perfectly the crash. Then on\n",
      "December 11, 1929, Murlyn Corporation—this was for another daughter—bought\n",
      "42,506 shares of stock from an affiliate of the Chase National Bank and financed\n",
      "this purchase with a loan of $6,588,430 from the Chase National Bank and from\n",
      "the Shermar Corporation. These shares were used to cover Shermar's short sale,\n",
      "i.e., to repay the loan of securities. The profits on the operation—at a time\n",
      "when many other people were doing much, much less well—was $4,008,538.7 People\n",
      "of carping tendencies might hold the profit was earned by the bank, whose stock\n",
      "it was, whose officer Wiggin was, and which had provided the money for the\n",
      "operation. In fact, the gain all went to Wiggin. Mr. Wiggin subsequently\n",
      "defended loans by banks to their own officers to allow them to speculate in\n",
      "their own stock on the grounds that it developed an interest in their\n",
      "institution. However, by this fine of reasoning, loans to finance short sales\n",
      "present a difficulty: presumably they develop an interest in having the\n",
      "institution, and hence its stock, behave as badly as possible. Pressed on this\n",
      "point, Mr. Wiggin expressed doubt as to whether officers should sell their own\n",
      "companies short. At the end of 1932 Mr. Wiggin requested that he not be re-\n",
      "elected Chairman of the Governing Board of the bank. He was approaching sixty-\n",
      "five, and he noted with some slight overstatement that his \"heart and energies\n",
      "[had] been concentrated for many years in promoting the growth, welfare, and\n",
      "usefulness of the Chase National Bank.\"8 It also seems probable that Winthrop W.\n",
      "Aldrich, who had come into the Chase as the result of a merger with the\n",
      "Equitable Trust Company and who represented a more austere tradition in\n",
      "commercial banking—the Equitable was controlled by the Rockefellers—had come to\n",
      "regard Mr. Wiggin as dispensable.9 The Executive Committee of the Chase, \"in\n",
      "order to discharge in some measure the obligations of this bank to Mr.\n",
      "Wiggin,\"10 by unanimous action voted him a life salary of $100,000. It was later\n",
      "brought out that this gesture of inspired generosity had been the impulse of Mr.\n",
      "Wiggin himself. In the months following Mr. Wiggin's retirement his activities\n",
      "became a matter of detailed study by a Senate committee. Mr. Aldrich, his\n",
      "successor, confessed his surprise at the extent and diversity of his\n",
      "predecessor's enterprises and said that the voting of the life salary was a\n",
      "terrible mistake. Mr. Wiggin later renounced the compensation.         IIIBy\n",
      "comparison with the National City the troubles of the Chase were slight. Mr.\n",
      "Wiggin was a reserved, some described him as a rather scholarly, man. The head\n",
      "of the National City, Charles E. Mitchell, on the other hand, was a genial\n",
      "extrovert with a talent for headlines. He was known to one and all as a leading\n",
      "prophet of the New Era.In the autumn of 1929 there were rumors in Wall Street\n",
      "that Mitchell would resign. He did not, and the rumors were described by Percy\n",
      "A. Rockefeller, an associate in numerous rather fervent stock market operations\n",
      "and a director of the bank, as \"too absurd to be considered by any sensible\n",
      "person.\"11 For the next two or three years Mitchell was rather out of the news.\n",
      "Then at nine o'clock on the evening of March 21, 1933, he was arrested by\n",
      "Assistant U.S. District Attorney Thomas E. Dewey and charged with evasion of\n",
      "income taxes. Many of the facts were never seriously in dispute. Like Wiggin,\n",
      "Mitchell had been operating extensively in the stock of his own bank, although\n",
      "possibly for more defensible reasons. Nineteen-twenty-nine was a year of bank\n",
      "mergers, and Charles E. Mitchell was no man to resist a trend. By early autumn\n",
      "of 1929 he had all but completed a merger with the Com Exchange Bank. The\n",
      "directors of the two institutions had approved; all that remained was the\n",
      "formality of ratification by the stockholders. Holders of Com Exchange stock\n",
      "were to receive, at their option, four-fifths of a share of National City stock\n",
      "or $360 in cash. The price of National City stock was then above 500, so it was\n",
      "certain that the Com Exchange stockholders would take the stock.Then came the\n",
      "crash. The price of National City stock dropped to around 425, and at any price\n",
      "below 450—four-fifths of which equalled the $360 in cash—the stockholders of the\n",
      "Com Exchange would take money. To buy out all the Com Exchange stockholders for\n",
      "cash would cost the National City around $200 million. That was too much, so\n",
      "Mitchell undertook to save the deal. He began buying National City stock, and\n",
      "during the week of October 28 he arranged to borrow twelve million dollars from\n",
      "J. P. Morgan and Company with which to buy more. (Twelve million was a sizable\n",
      "sum both for Mitchell and for Morgan's, even at that time. Only ten million was\n",
      "actually used, and of this four million was repaid within a week or so. Possibly\n",
      "some of the Morgan partners had second thoughts on the wisdom of the loan.)The\n",
      "coup failed. Like so many others, Mitchell learned how different it was to\n",
      "support a stock when everyone wanted to sell as compared with those days but a\n",
      "few weeks back when everyone wanted to buy. The price of National City stock\n",
      "sank lower and lower. Mitchell reached the end of his resources and gave up.\n",
      "This was no time for false pride, and with some mild prodding from the\n",
      "management, the National City stockholders repudiated that management and\n",
      "rejected the now disastrous deal. Mitchell, however, was left with a formidable\n",
      "debt to J. P. Morgan and Company. This debt was secured by the stock that had\n",
      "been purchased to support the market and by Mitchell's personal holdings, but\n",
      "its value was shrinking grievously. By the end of the year National City stock\n",
      "was near 200, down from more than 500, and close to the value at which Morgan's\n",
      "had accepted it as collateral. Now Mitchell faced another misfortune, or,\n",
      "rather, an earlier piece of good fortune now became a disaster. As an executive\n",
      "of the National City Bank, Mitchell's pay was a modest $25,000. However, the\n",
      "bank had an incentive system which may still hold some sort of record for\n",
      "munificence. After a deduction of 8 per cent, 20 per cent of the profits of the\n",
      "bank and of its security affiliate, the National City Company, were paid into a\n",
      "management fund. This was divided twice a year between the principal officers by\n",
      "an arrangement which must have made for an interesting half hour. Each officer\n",
      "first dropped in a hat an unsigned ballot suggesting the share of the fund that\n",
      "Chairman Mitchell should have. Then each signed a ballot giving his estimate of\n",
      "the worth of each of the other eligible officers, himself excluded. The average\n",
      "of these estimates guided the Executive Committee of the bank in fixing the\n",
      "percentages of the fund each officer was to have.The years 1928 and 1929 were a\n",
      "time of excellent profits. Mitchell's subordinates had also taken a favorable\n",
      "view of his work. For the full year 1928 his cut was $1,316,634.14. 1929 was\n",
      "even better. The division at the end of the first half of that year brought him\n",
      "no less than $1,108,000.12 Dividends and numerous other activities had further\n",
      "augmented his income, and all of this meant a serious tax liability. It would\n",
      "have been easy to sell some National City stock and establish a tax loss, but,\n",
      "as noted, the stock was pledged with J. P. Morgan and Company. Nevertheless,\n",
      "Mitchell sold the stock—to his wife: 18,300 shares were disposed of to this\n",
      "possibly unsuspecting lady at 212, for the exceedingly satisfying loss of\n",
      "$2,872,305.50. This wiped out all tax liability for 1929. Morgan's was not, it\n",
      "appears, notified of the change of ownership of the stock they held. Somewhat\n",
      "later Mitchell reacquired the stock from his wife, also at a price of 212.\n",
      "Before then there had been a further sickening slide in the price, and had\n",
      "Chairman Mitchell bought the stock in the open market rather than from his wife,\n",
      "he could have got it for around 40. Asked about the transaction by Senator\n",
      "Brookhart of Iowa during a Senate hearing, Mitchell, in a burst of candor that\n",
      "must have devastated his lawyer, said: \"I sold this stock, frankly, for tax\n",
      "purposes.\"13 This frankness led directly to his indictment a few weeks\n",
      "later.Following his testimony, Mitchell had resigned from the National City\n",
      "Bank. His trial in New York during May and June of 1933 was something of a\n",
      "sensation, although the headlines were necessarily subordinate to the larger\n",
      "ones currently being made in Washington. In his inaugural address on March 4,\n",
      "Roosevelt had promised to drive the money-changers from the temple. Mitchell was\n",
      "widely regarded as the first.On June 22, Mitchell was acquitted by the jury on\n",
      "all counts. The sales as required by the tax laws were held to be bona fide\n",
      "transactions made in good faith. The Times reporter covering the trial thought\n",
      "that both Mitchell and his lawyer received the verdict with surprise. Attorney\n",
      "General Cummings said that he still believed in the jury system. Mitchell later\n",
      "resumed his career in Wall Street as head of Blyth and Company. The government\n",
      "entered a civil claim for the taxes and won a judgment of $1,100,000 in taxes\n",
      "and penalties. Mitchell appealed the case through to the Supreme Court, lost,\n",
      "and made a final settlement with the government on December 27, 1938. On his\n",
      "behalf it must be stressed that the device by which he sought to reduce his tax\n",
      "liability was far more common then than now. The Senate investigations of 1933\n",
      "and 1934 showed that tax avoidance had brought individuals of the highest\n",
      "respectability into extraordinary financial intercourse with their wives.14\n",
      "IVOur political tradition sets great store by the generalized symbol of evil.\n",
      "This is the wrongdoer whose wrongdoing will be taken by the public to be the\n",
      "secret propensity of a whole community or class. We search avidly for such\n",
      "people, not so much because we wish to see them exposed and punished as\n",
      "individuals, but because we cherish the resulting political discomfort of their\n",
      "friends. To uncover an evil man among the friends of one's foes had long been a\n",
      "recognized method of advancing one's political fortunes. However, in recent\n",
      "times the technique has been greatly improved and refined by the added firmness\n",
      "with which the evil of the evildoer is now attributed to friends, acquaintances,\n",
      "and all who share his way of life. In the nineteen-thirties Wall Street was\n",
      "exceptionally well endowed with enemies. There were some socialists and\n",
      "communists who believed that capitalism should be abolished and obviously did\n",
      "not seek to have its citadel preserved. There were some people who merely\n",
      "thought that Wall Street was bad. There were yet others who did not seek to have\n",
      "Wall Street abolished or who did not care much about its allegedly evil ways but\n",
      "who enjoyed as a matter of course the discomfiture of the rich and the powerful\n",
      "and the proud. There were those who had lost money in Wall Street. Most of all\n",
      "there was the New Deal. The administrations of Coolidge and Hoover had had an\n",
      "extremely overt alliance with the great financial interests which Wall Street\n",
      "symbolized. With the advent of the New Deal the sins of Wall Street became the\n",
      "sins of the political enemy. What was bad for Wall Street was bad for the\n",
      "Republican Party.For anyone who was in search of symbolic evil in Wall Street—of\n",
      "individuals whose misbehavior would stigmatize the whole community—the discovery\n",
      "that the heads of the National City and Chase had been guilty of grave lapses\n",
      "would seem to be almost ideal. These were the two best-known and most\n",
      "influential banks; what could have been better than default here?That the\n",
      "shortcomings of Mr. Wiggin and Mr. Mitchell were much welcomed is, of course,\n",
      "clear. Yet in some indefinable sense they were not of that part of Wall Street\n",
      "that people suspected most. Wall Street's crime, in the eyes of its classical\n",
      "enemies, was less its power than its morals. And the center of immorality was\n",
      "not the banks but the stock market. It was on the stock market that men gambled\n",
      "not alone with their own money, but with the wealth of the country. The stock\n",
      "market, with its promise of easy riches, was what led good if not very wise men\n",
      "to perdition—like the cashier of the local bank who was also a vestryman. The\n",
      "senseless gyrations of the stock market affected farm prices and land values and\n",
      "the renewal of notes and mortgages. Though to the sophisticated radical the\n",
      "banks might be the real menace, sound populist attitudes pointed the finger of\n",
      "suspicion at the New York Stock Exchange. There, accordingly, was the place, if\n",
      "possible, to find the symbol of evil, for there was the institution about which\n",
      "people were ready to believe the worst. The search for a really adequate\n",
      "miscreant in the Stock Exchange began in April of 1932. The task was undertaken\n",
      "by the Senate Committee on Banking and Currency (later by a subcommittee) and\n",
      "its instructions, graced by the usual split infinitive, were \"to thoroughly\n",
      "investigate practices of stock exchanges...\" Under the later guidance of\n",
      "Ferdinand Pecora, this committee became the scourge of commercial, investment,\n",
      "and private bankers. But this was not foreseen when it was organized. The\n",
      "original and more or less exclusive object of the inquiry was the market for\n",
      "securities.On the whole, this part of the investigation was unproductive. The\n",
      "first witness, when the hearings opened on April 11, 1932, was Richard\n",
      "Whitney.15 On November 30, 1929, the Governing Committee of the New York Stock\n",
      "Exchange had passed a resolution of appreciation for the \"efficient and\n",
      "conscientious\" labors of their acting president during the recent storm. It is\n",
      "an \"old saying,\" the Resolution had stated, \"that great emergencies produce the\n",
      "men who are competent to deal with them...\" This sense of indebtedness made it\n",
      "inevitable that when Edward H. H. Simmons retired as President of the Exchange\n",
      "in 1930 after six years in office, Whitney would be elected to succeed him. As\n",
      "President of the Exchange it thus fell to Whitney, in the spring of 1932, to\n",
      "assume the task of protecting the stock market from its critics. Whitney was not\n",
      "in all respects an ingratiating witness. One of his successors in office not\n",
      "long ago compared his general manner and bearing with that of Secretary of\n",
      "Defense Charles E. Wilson at the hearings on his confirmation as Secretary of\n",
      "Defense in early 1953. Whitney admitted to no serious fault in the past\n",
      "operations of the Exchange or even to the possibility of error. He supplied the\n",
      "information that was requested, but he was not unduly helpful to senators who\n",
      "sought to penetrate the mysteries of short selling, sales against the box,\n",
      "options, pools, and syndicates. He seemed to feel that these things were beyond\n",
      "the senators' intelligence. Alternatively he implied that they were things that\n",
      "every intelligent schoolboy understood and it was painful for him to have to go\n",
      "over the obvious. He was so unwise as to get into a discussion of personal\n",
      "economic philosophy with Senator Smith W. Brookhart of Iowa, one of the\n",
      "committee members who believed, devoutly, that the Exchange was the particular\n",
      "invention of the devil. The government, not Wall Street, was responsible for the\n",
      "current bad times, Whitney averred, and the government, he believed, could make\n",
      "its greatest contribution to recovery by balancing the budget and thus restoring\n",
      "confidence. To balance the budget he recommended cutting the pensions and\n",
      "benefits of veterans who had no service-connected disability and also all\n",
      "government salaries. When asked about cutting his own pay he said no—it was\n",
      "\"very little.\" Pressed for the amount, he said that currently it was only about\n",
      "$60,000. His attention was drawn by the committee members to the fact that this\n",
      "was six times what a senator received, but Whitney remained adamantly in favor\n",
      "of cutting the public pay, including that of senators.16 In spite of Whitney's\n",
      "manner, or possibly because of it, several days of questioning produced little\n",
      "evidence of wrongdoing and no identification of wrongdoers. Prior to the crash\n",
      "Whitney had heard generally of syndicates and pools, but he could give no\n",
      "details. He repeatedly assured the committee that the Exchange had these and\n",
      "other matters well under control. He took exception to Senator Brookhart's\n",
      "contention that the market was a gambling hell and should be padlocked. In the\n",
      "end Whitney was excused before he had quite completed his testimony.When the\n",
      "interrogation of Whitney showed clear signs of being unproductive, the committee\n",
      "turned to the famous market operators. These, too, were disappointing. All that\n",
      "could be proved was what everyone knew, namely, that Bernard E. (\"Sell 'em Ben\")\n",
      "Smith, M. J. Meehan, Arthur W. Cutten, Harry F. Sinclair, Percy A. Rockefeller,\n",
      "and others had been engaged in large-scale efforts to rig the market. Harry F.\n",
      "Sinclair, for example, was shown to have engaged in especially extensive\n",
      "operations in Sinclair Consolidated Oil. This was much like identifying William\n",
      "Z. Foster with the Communist Party. It was impossible to imagine Harry Sinclair\n",
      "not being involved in some intricate maneuver in high finance. Moreover,\n",
      "reprehensible as these activities were, it remained that only three short years\n",
      "before they had been regarded with breathless admiration. The problem here was\n",
      "somewhat similar to that encountered in the great red-hunt of the latter\n",
      "forties. Then there was constant embarrassment over the short time that had\n",
      "elapsed since Red Russia had been our gallant Soviet ally. It is true that the\n",
      "big operators, as they appeared on the stand, were not an especially\n",
      "prepossessing group. As noted earlier, Arthur Cutten's memory was extremely\n",
      "defective. M. J. Meehan was in bad health and mistakenly went abroad when he was\n",
      "supposed to go to Washington. (He later apologized handsomely for the error.)\n",
      "Few of the others could remember much about their operations, Napoleonic though\n",
      "they had once seemed. But men cannot be brought to trial for being\n",
      "unprepossessing. And the dubious demeanor and bad memories of the market\n",
      "operators did not directly involve the reputation of the New York Stock\n",
      "Exchange. It is possible to have a poor view of touts, tipsters, and bookies\n",
      "without thinking the worse of Churchill Downs.In earlier times of trouble on the\n",
      "stock market, stock exchange firms had failed, on occasion by the score. In the\n",
      "fall of 1929 the failures were unimportant. In the first week of the crash no\n",
      "member firm of the New York Stock Exchange had to suspend; only one smallish\n",
      "member firm went under during the period of the panic. There were some\n",
      "complaints by customers of mistreatment. But there were more customers who,\n",
      "during the worst days, were carried by their brokers after their margins had\n",
      "been impaired or depleted. The standards of commercial morality of the members\n",
      "of the Exchange would seem to have been well up to the average of the late\n",
      "twenties. They may have been much more rigorous. This would seem to be the most\n",
      "obvious explanation why the Exchange and its members survived so well the\n",
      "investigations of the thirties. They did not come through unscathed, but they\n",
      "suffered no obloquy comparable, say, with that of the great bankers. In the\n",
      "congressional investigations no flagrant miscreant of any kind was uncovered on\n",
      "the Exchange to serve as the symbolic bad apple. Then, on March 10, 1938,\n",
      "District Attorney Thomas E. Dewey—who had arrested Charles E. Mitchell and who\n",
      "somehow has escaped a reputation as the nemesis of Wall Street—ordered the\n",
      "arraignment of Richard Whitney. The charge was grand larceny.         VThe rush\n",
      "to get in on the act, to use the recent idiom, when Whitney was arrested is a\n",
      "measure of the yearning for a malefactor in the stock market. It can be compared\n",
      "only with the stampede which followed the announcement by Attorney General\n",
      "Herbert Brownell in the autumn of 1953 that former President Truman had shielded\n",
      "treason. On the day following his first arrest, Whitney was arrested again by\n",
      "New York State Attorney General John J. Bennett. Mr. Bennett had been conducting\n",
      "an investigation of Whitney's affairs, and he bitterly accused Mr. Dewey of\n",
      "legal claim-jumping. In the next few weeks virtually every public body or\n",
      "tribunal with a plausible excuse for doing so, called Whitney to enlarge on his\n",
      "wrongdoing.The detailed story of Richard Whitney's misfortunes do not belong to\n",
      "this chronicle. Many of them occurred after the period with which this history\n",
      "is concerned. There is need here to cover only those operations that were deemed\n",
      "to implicate the market.Whitney's dishonesty was of a casual, rather juvenile\n",
      "sort. Associates of the day have since explained it as the result of an\n",
      "unfortunate failure to realize that the rules, which were meant for other\n",
      "people, also applied to him. Much more striking than Whitney's dishonesty was\n",
      "the clear fact that he was one of the most disastrous businessmen in modern\n",
      "history. Theft was almost a minor incident pertaining to his business\n",
      "misfortunes. In the twenties the Wall Street firm of Richard Whitney and Company\n",
      "was an unspectacular bond house with a modest business. Whitney apparently felt\n",
      "that it provided insufficient scope for his imagination, and with the passing\n",
      "years he moved on to other enterprises, including the mining of mineral colloids\n",
      "and the marketing of peat humus in Florida. He had also become interested in the\n",
      "distilling of alcoholic beverages, mainly applejack, in New Jersey. Nothing is\n",
      "so voracious as a losing business, and eventually Whitney had three of them. To\n",
      "keep them going he borrowed from banks, investment bankers, other stock exchange\n",
      "members, and heavily from his brother, George Whitney, a partner of J. P. Morgan\n",
      "and Company. The loans so negotiated, from the early twenties on, totaled in the\n",
      "millions, many of them unsecured. As time passed Whitney was increasingly\n",
      "pressed. When one loan became due he was forced to replace it with another and\n",
      "to borrow still more for the interest on those outstanding. Beginning in 1933\n",
      "his stock exchange firm was insolvent, although this did not become evident for\n",
      "some five years.17Finally, like so many others, Richard Whitney learned the cost\n",
      "of supporting a stock on a falling market. In 1933, Richard Whitney and\n",
      "Company—the affairs of Whitney and his company were almost completely\n",
      "indistinguishable—had invested in between ten and fifteen thousand shares of\n",
      "Distilled Liquors Corporation, the New Jersey manufacturer of applejack and\n",
      "other intoxicants. The price was $15 a share. In the spring of 1934 the stock\n",
      "reached 45 in over-the-counter trading. In January 1935 it was fisted on the New\n",
      "York Curb Exchange. Inevitably Whitney posted the stock as collateral for\n",
      "various of his loans. Unhappily, popular enthusiasm for the products of the\n",
      "firm, even in the undiscriminating days following repeal, was remarkably slight.\n",
      "The firm made no money and by June 1936 the price of the stock was down to 11.\n",
      "This drop had a disastrous effect on its value as collateral, and the unhappy\n",
      "Whitney tried to maintain the value by buying more of it. (He later made the\n",
      "claim that he wanted to provide the other investors in the company with a market\n",
      "for their stock,18 which if true meant that he was engaging in one of the most\n",
      "selfless acts since the death of Sydney Carton.) All the other investors\n",
      "unloaded on Whitney. At the time of his failure, of the 148,750 shares\n",
      "outstanding, Whitney or his firm owned 137,672. By then the value had dropped to\n",
      "between three and four dollars a share. Mention has been made of the tendency of\n",
      "people in this period to swindle themselves. Whitney, in his effort to support\n",
      "the stock of Distilled Liquors Corporation, unquestionably emerged as the Ponzi\n",
      "of financial self-deception. As the result of his operation he had all his old\n",
      "debts, many new ones incurred in supporting the stock, all the stock—and the\n",
      "stock was nearly worthless.As his position became more complex, Richard Whitney\n",
      "resorted increasingly to an expedient which he had been using for several\n",
      "years—that of posting securities belonging to other people which were in his\n",
      "custody as collateral for his loans. By early 1938 he had reached the end of a\n",
      "surprising capacity to borrow money. Late in the preceding autumn he had had a\n",
      "large last loan from his brother to release securities belonging to the gratuity\n",
      "fund of the Stock Exchange—a fund out of which payments were made on the death\n",
      "of members—which he had appropriated and pledged for a bank loan. He was now\n",
      "desperately, almost pathetically, visiting the most casual of acquaintances in\n",
      "search of funds. The rumor spread that he was in poor condition. Still, on March\n",
      "8, there was a stunned silence on the floor of the Exchange when President\n",
      "Charles R. Gay announced from the rostrum the suspension of Richard Whitney and\n",
      "Company for insolvency. Members were rather more aghast when they learned that\n",
      "Whitney had been engaged in theft on a large scale for a long period.With no\n",
      "small dignity Whitney made a full disclosure of his operations, refused to enter\n",
      "any sort of plea in his own defense, and passed permanently from sight.\n",
      "VIThe failure of the smallest country banker caused more personal hardship,\n",
      "anguish, and privation than the insolvency of Richard Whitney. His victims were\n",
      "almost uniquely able to afford their loss. And the sums which he stole, while\n",
      "substantial, did not place him in the ranks of the great defaulters of the\n",
      "day—they would not have paid the interest on Ivar Kreuger's larceny for a year.\n",
      "Yet, from the point of view of the antagonists of Wall Street, his default was\n",
      "ideal. Rarely has a crime been more joyously received.Whitney's identification\n",
      "was wholly with the Stock Exchange, the symbolic center of sin. Moreover, he had\n",
      "been its president and its uncompromising defender before the Congress and the\n",
      "public in its period of trial. He was a Republican, an arch-conservative, and\n",
      "was loosely associated in the financial community with J. P. Morgan and Company.\n",
      "He had himself taken a strong position in favor of honesty. Speaking in St.\n",
      "Louis in 1932, at a time when his own larceny was already well advanced, Whitney\n",
      "had said sternly that one of the prime necessities \"of a great market is that\n",
      "brokers must be honest and financially responsible.\" He looked forward to the\n",
      "day when the financial supervision by the Exchange of its members would be so\n",
      "strict that failure \"will be next to impossible.\"19 Finally, even by his\n",
      "colleagues, Whitney was regarded as a trifle upstage. In his last days he had\n",
      "been reduced to the ultimate indignity of trying to borrow money from the market\n",
      "operator, Bernard E. Smith. Smith, at best a lower middle-brow figure, later\n",
      "told a Securities and Exchange Commission examiner: \"He came up to see me and\n",
      "said he would like to get this over quickly, and told me he would like to borrow\n",
      "$250,000 on his face. I remarked he was putting a pretty high value on his face,\n",
      "so he said ... his back was to the wall and he had to have $250,000. I told him\n",
      "he had a lot of nerve to ask me for $250,000 when he didn't even bid me the time\n",
      "of day. I told him I frankly didn't like him—that I wouldn't loan him a dime.\"20\n",
      "On any free vote for the man best qualified to bring discredit to Wall Street,\n",
      "Whitney would have won by a wide margin.The parallel between Whitney and a more\n",
      "recent culprit is interesting. During the thirties the New Dealers were\n",
      "exuberantly uncovering the financial derelictions of their opposition. (It is\n",
      "interesting that dishonesty and not the more orthodox offenses of capitalism\n",
      "like abuse of power or the exploitation of the people were in these days the\n",
      "nemesis of conservatives.) In the nineteen-forties and fifties Republicans, as\n",
      "avidly, were discovering that there were New Dealers who had been communists.\n",
      "Thus it came about that a decade later the counterpart of Richard Whitney was\n",
      "Alger Hiss.Each served admirably the enemies of his class. Each in origin,\n",
      "education, associations, and career pretension epitomized that class. In each\n",
      "case the first reaction of friends to the allegations of guilt was disbelief.\n",
      "Whitney's past role in his community had been more prominent, and hence from the\n",
      "viewpoint of his enemies he was a more satisfactory figure than Alger Hiss. In\n",
      "the government hierarchy, Hiss was a distinctly routine figure. His eminence as\n",
      "a global statesman was synthesized ex post facto and he also gained much\n",
      "prominence during two long trials. Whitney, with no fanfare, accepted his\n",
      "fate.There is, perhaps, a moral worth drawing from the careers of Whitney and\n",
      "Hiss. Neither the fact that Whitney was convicted of purloining securities nor\n",
      "that Hiss purloined documents is convincing proof that their friends,\n",
      "associates, and contemporaries were doing the same. On the contrary, the\n",
      "evidence would indicate that most brokers were honest as a matter of absolute\n",
      "routine, and most New Dealers, so far from being in league with the Russians,\n",
      "wished only that they might be invited once to taste caviar at the Soviet\n",
      "embassy. Both liberals and conservatives, left and right, have now had personal\n",
      "experience with the use of the symbolic evil. The injustice of the device is\n",
      "evident. What may be a more compelling point, so are its dangers. In accordance\n",
      "with an old but not outworn tradition, it might now be wise for all to conclude\n",
      "that crime, or even misbehavior, is the act of an individual, not the\n",
      "predisposition of a class.         VIIThe Whitney affair brought a marked change\n",
      "in the relations between the Exchange and the federal government, and, in some\n",
      "measure, between the Exchange and the general public. In the Securities Act of\n",
      "1933, and more comprehensively in the Securities Exchange Act of 1934, the\n",
      "government had sought to prohibit some of the more spectacular extravagances of\n",
      "1928 and 1929. Full disclosure was required on new security issues, although no\n",
      "way was found of making would-be investors read what was disclosed. Inside\n",
      "operations and short selling after the maimer of Mr. Wiggin were outlawed.\n",
      "Authority was given to the Federal Reserve Board to fix margin requirements and\n",
      "these could, if necessary, be made 100 per cent and thus eliminate margin\n",
      "trading entirely. Pool operations, wash sales, the dissemination of tips or\n",
      "patently false information and other devices for rigging or manipulating the\n",
      "market were prohibited. Commercial banks were divorced from their securities\n",
      "affiliates. Most important, the principle was enunciated that the New York Stock\n",
      "Exchange and the other exchanges were subject to public regulation and the\n",
      "Securities and Exchange Commission was established to apply and enforce such\n",
      "regulation. This was somewhat bitter medicine. Moreover, regulatory bodies, like\n",
      "the people who comprise them, have a marked life cycle. In youth they are\n",
      "vigorous, aggressive, evangelistic, and even intolerant. Later they mellow, and\n",
      "in old age—after a matter of ten or fifteen years—they become, with some\n",
      "exceptions, either an arm of the industry they are regulating or senile. The SEC\n",
      "was especially aggressive. To any young regulatory body, after all, Wall Street\n",
      "was certain to seem a challenging antagonist.Until the Whitney affair Wall\n",
      "Street—always with exceptions—was disposed to fight back. It insisted on the\n",
      "right of a financial community in general, and of a securities market in\n",
      "particular, to conduct its affairs in its own way, by its own lights and to\n",
      "govern itself. On the evening before the suspension of Whitney was announced\n",
      "from the rostrum, Charles R. Gay, the President of the Exchange, and Howland S.\n",
      "Davis, Chairman of the Committee on Business Conduct—Whitney had been a\n",
      "predecessor of both in the two offices—made their way to Washington. There they\n",
      "reported their unhappy news to William O. Douglas and John W. Hanes of the SEC.\n",
      "The trip, in far more than a symbolic sense, represented the surrender of the\n",
      "Exchange. The cold war over regulation came to an end and was not thereafter\n",
      "resumed. While the Whitney default confirmed the victory of the New Deal on the\n",
      "issue of regulation and also served admirably to confirm the more general\n",
      "suspicion of moral delinquency in downtown New York, it was Wall Street's good\n",
      "fortune that it came late. By 1938 the New Deal assault on big business was on\n",
      "the wane; some leaders of the original shock troops were already polishing up\n",
      "speeches on the virtues of the free enterprise system. By then, also, it was\n",
      "accepted New Deal theology that all necessary economic reforms had been revealed\n",
      "and those that had not been enacted were on request from Congress. No further\n",
      "reforms of the securities markets of any importance were on the agenda.\n",
      "Henceforth Wall Street looked ingratiatingly at Washington and Washington merely\n",
      "looked blank.         CHAPTER IXCause and ConsequenceAFTER THE Great Crash came\n",
      "the Great Depression which lasted, with varying severity, for ten years. In\n",
      "1933, Gross National Product (total production of the economy) was nearly a\n",
      "third less than in 1929. Not until 1937 did the physical volume of production\n",
      "recover to the levels of 1929, and then it promptly slipped back again. Until\n",
      "1941 the dollar value of production remained below 1929. Between 1930 and 1940\n",
      "only once, in 1937, did the average number unemployed during the year drop below\n",
      "eight million. In 1933 nearly thirteen million were out of work, or about one in\n",
      "every four in the labor force. In 1938 one person in five was still out of\n",
      "work.1It was dining this dreary time that 1929 became a year of myth. People\n",
      "hoped that the country might get back to twenty-nine; in some industries or\n",
      "towns when business was phenomenally good it was almost as good as in twenty-\n",
      "nine; men of outstanding vision, on occasions of exceptional solemnity, were\n",
      "heard to say that 1929 \"was no better than Americans deserve.\"On the whole, the\n",
      "great stock market crash can be much more readily explained than the depression\n",
      "that followed it. And among the problems involved in assessing the causes of\n",
      "depression none is more intractable than the responsibility to be assigned to\n",
      "the stock market crash. Economics still does not allow final answers on these\n",
      "matters. But, as usual, something can be said.         IIAs already so often\n",
      "emphasized, the collapse in the stock market in the autumn of 1929 was implicit\n",
      "in the speculation that went before. The only question concerning that\n",
      "speculation was how long it would last. Sometime, sooner or later, confidence in\n",
      "the short-run reality of increasing common stock values would weaken. When this\n",
      "happened, some people would sell, and this would destroy the reality of\n",
      "increasing values. Holding for an increase would now become meaningless; the new\n",
      "reality would be falling prices. There would be a rush, pellmell, to unload.\n",
      "This was the way past speculative orgies had ended. It was the way the end came\n",
      "in 1929. It is the way speculation will end in the future.We do not know why a\n",
      "great speculative orgy occurred in 1928 and 1929. The long accepted explanation\n",
      "that credit was easy and so people were impelled to borrow money to buy common\n",
      "stocks on margin is obviously nonsense. On numerous occasions before and since\n",
      "credit has been easy, and there has been no speculation whatever. Furthermore,\n",
      "much of the 1928 and 1929 speculation occurred on money borrowed at interest\n",
      "rates which for years before, and in any period since, would have been\n",
      "considered exceptionally astringent. Money, by the ordinary tests, was tight in\n",
      "the late twenties.Far more important than rate of interest and the supply of\n",
      "credit is the mood. Speculation on a large scale requires a pervasive sense of\n",
      "confidence and optimism an' conviction that ordinary people were meant to be\n",
      "rich. People must also have faith in the good intentions and even in the\n",
      "benevolence of others, for it is by the agency of others that they will get\n",
      "rich. In 1929 Professor Dice observed: \"The common folks believe in their\n",
      "leaders. We no longer look upon the captains of industry as magnified crooks.\n",
      "Have we not heard their voices over the radio? Are we not familiar with their\n",
      "thoughts, ambitions, and ideals as they have expressed them to us almost as a\n",
      "man talks to his friend?\"2 Such a feeling of trust is essential for a boom. When\n",
      "people are cautious, questioning, misanthropic, suspicious, or mean, they are\n",
      "immune to speculative enthusiasms. Savings must also be plentiful. Speculation,\n",
      "however it may rely on borrowed funds, must be nourished in part by those who\n",
      "participate. If savings are growing rapidly, people will place a lower marginal\n",
      "value on their accumulation; they will be willing to risk some of it against the\n",
      "prospect of a greatly enhanced return. Speculation, accordingly, is most likely\n",
      "to break out after a substantial period of prosperity, rather than in the early\n",
      "phases of recovery from a depression. Macaulay noted that between the\n",
      "Restoration and the Glorious Revolution Englishmen were at loss to know what to\n",
      "do with their savings and that the \"natural effect of this state of things was\n",
      "that a crowd of projectors, ingenious and absurd, honest and knavish, employed\n",
      "themselves in devising new schemes for the employment of redundant capital.\"\n",
      "Bagehot and others have attributed the South Sea Bubble to roughly the same\n",
      "causes.3 In 1720 England had enjoyed a long period of prosperity, enhanced in\n",
      "part by war expenditures, and during this time private savings are believed to\n",
      "have grown at an unprecedented rate. Investment outlets were also few and\n",
      "returns low. Accordingly, Englishmen were anxious to place their savings at the\n",
      "disposal of the new enterprises and were quick to believe that the prospects\n",
      "were not fantastic. So it was in 1928 and 1929.Finally, a speculative outbreak\n",
      "has a greater or less immunizing effect. The ensuing collapse automatically\n",
      "destroys the very mood speculation requires. It follows that an outbreak of\n",
      "speculation provides a reasonable assurance that another outbreak will not\n",
      "immediately occur. With time and the dimming of memory, the immunity wears off.\n",
      "A recurrence becomes possible. Nothing would have induced Americans to launch a\n",
      "speculative adventure in the stock market in 1935. By 1955 the chances are very\n",
      "much better.         IIIAs noted, it is easier to account for the boom and crash\n",
      "in the market than to explain their bearing on the depression which followed.\n",
      "The causes of the Great Depression are still far from certain. A lack of\n",
      "certainty, it may also be observed, is not evident in the contemporary writing\n",
      "on the subject. Much of it tells what went wrong and why with marked firmness.\n",
      "However, this paradoxically can itself be an indication of uncertainty. When\n",
      "people are least sure they are often most dogmatic. We do not know what the\n",
      "Russians intend, so we state with great assurance what they will do. We\n",
      "compensate for our inability to foretell the consequences of, say, rearming\n",
      "Germany by asserting positively just what the consequences will be. So it is in\n",
      "economics. Yet, in explaining what happened in 1929 and after, one can\n",
      "distinguish between explanations that might be right and those that are clearly\n",
      "wrong. A great many people have always felt that a depression was inevitable in\n",
      "the thirties. There had been (at least) seven good years; now by an occult or\n",
      "biblical law of compensation there would have to be seven bad ones. Perhaps,\n",
      "consciously or unconsciously, an argument that was valid for the stock market\n",
      "was brought to bear on the economy in general. Because the market took leave of\n",
      "reality in 1928 and 1929, it had at some time to make a return to reality. The\n",
      "disenchantment was bound to be as painful as the illusions were beguiling.\n",
      "Similarly, the New Era prosperity would some day evaporate; in its wake would\n",
      "come the compensating hardship.There is also the slightly more subtle conviction\n",
      "that economic life is governed by an inevitable rhythm. After a certain time\n",
      "prosperity destroys itself and depression corrects itself. In 1929 prosperity,\n",
      "in accordance with the dictates of the business cycle, had run its course. This\n",
      "was the faith confessed by the members of the Harvard Economic Society in the\n",
      "spring of 1929 when they concluded that a recession was somehow overdue.Neither\n",
      "of these beliefs can be seriously supported. The twenties by being comparatively\n",
      "prosperous established no imperative that the thirties be depressed. In the\n",
      "past, good times have given way to less good times and less good or bad to good.\n",
      "But change is normal in a capitalist economy. The degree of regularity in such\n",
      "movements is not great, though often thought to be.4 No inevitable rhythm\n",
      "required the collapse and stagnation of 1930–40.Nor was the economy of the\n",
      "United States in 1929 subject to such physical pressure or strain as the result\n",
      "of its past level of performance that a depression was bound to come. The notion\n",
      "that the economy requires occasional rest and resuscitation has a measure of\n",
      "plausibility and also a marked viability. During the summer of 1954 a\n",
      "professional economist on President Eisenhower's personal staff explained the\n",
      "then current recession by saying that the economy was enjoying a brief (and\n",
      "presumably well-merited) rest after the exceptional exertions of preceding\n",
      "years. In 1929 the labor force was not tired; it could have continued to produce\n",
      "indefinitely at the best 1929 rate. The capital plant of the country was not\n",
      "depleted. In the preceding years of prosperity, plant had been renewed and\n",
      "improved. In fact, depletion of the capital plant occurred during the ensuing\n",
      "years of idleness when new investment was sharply curtailed. Raw materials in\n",
      "1929 were ample for the current rate of production. Entrepreneurs were never\n",
      "more eupeptic. Obviously if men, materials, plant, and management were all\n",
      "capable of continued and even enlarged exertions a refreshing pause was not\n",
      "necessary. Finally, the high production of the twenties did not, as some have\n",
      "suggested, outrun the wants of the people. During these years people were indeed\n",
      "being supplied with an increasing volume of goods. But there is no evidence that\n",
      "their desire for automobiles, clothing, travel, recreation, or even food was\n",
      "sated. On the contrary, all subsequent evidence showed (given the income to\n",
      "spend) a capacity for a large further increase in consumption. A depression was\n",
      "not needed so that people's wants could catch up with their capacity to produce.\n",
      "IVWhat, then, are the plausible causes of the depression? The task of answering\n",
      "can be simplified somewhat by dividing the problem into two parts. First there\n",
      "is the question of why economic activity turned down in 1929. Second there is\n",
      "the vastly more important question of why, having started down, on this unhappy\n",
      "occasion it went down and down and down and remained low for a full decade.As\n",
      "noted, the Federal Reserve indexes of industrial activity and of factory\n",
      "production, the most comprehensive monthly measures of economic activity then\n",
      "available, reached a peak in June. They then turned down and continued to\n",
      "decline throughout the rest of the year. The turning point in other\n",
      "indicators—factory payrolls, freight-car loadings, and department store\n",
      "sales—came later, and it was October or after before the trend in all of them\n",
      "was clearly down. Still, as economists have generally insisted, and the matter\n",
      "has the high authority of the National Bureau of Economic Research,5 the economy\n",
      "had weakened in the early summer well before the crash.This weakening can be\n",
      "variously explained. Production of industrial products, for the moment, had\n",
      "outrun consumer and investment demand for them. The most likely reason is that\n",
      "business concerns, in the characteristic enthusiasm of good times, misjudged the\n",
      "prospective increase in demand and acquired larger inventories than they later\n",
      "found they needed. As a result they curtailed their buying, and this led to a\n",
      "cutback in production. In short, the summer of 1929 marked the beginning of the\n",
      "familiar inventory recession. The proof is not conclusive from the (by present\n",
      "standards) limited figures available. Department store inventories, for which\n",
      "figures are available, seem not to have been out of line early in the year. But\n",
      "a mild slump in department store sales in April could have been a signal for\n",
      "curtailment. Also there is a chance—one that students of the period have\n",
      "generally favored—that more deep-seated factors were at work and made themselves\n",
      "seriously evident for the first time during that summer. Throughout the twenties\n",
      "production and productivity per worker grew steadily: between 1919 and 1929,\n",
      "output per worker in manufacturing industries increased by about 43 per cent.6\n",
      "Wages, salaries, and prices all remained comparatively stable, or in any case\n",
      "underwent no comparable increase. Accordingly, costs fell and with prices the\n",
      "same, profits increased. These profits sustained the spending of the well-to-do,\n",
      "and they also nourished at least some of the expectations behind the stock\n",
      "market boom. Most of all they encouraged a very high level of capital\n",
      "investment. During the twenties, the production of capital goods increased at an\n",
      "average annual rate of 6.4 per cent a year; non-durable consumers' goods, a\n",
      "category which includes such objects of mass consumption as food and clothing,\n",
      "increased at a rate of only 2.8 per cent.7 (The rate of increase for durable\n",
      "consumers' goods such as cars, dwellings, home furnishings, and the like, much\n",
      "of it representing expenditures of the well-off to well-to-do, was 5.9 per\n",
      "cent.) A large and increasing investment in capital goods was, in other words, a\n",
      "principal device by which the profits were being spent.8 It follows that\n",
      "anything that interrupted the investment oudays—anything, indeed, which kept\n",
      "them from showing the necessary rate of increase—could cause trouble. When this\n",
      "occurred, compensation through an increase in consumer spending could not\n",
      "automatically be expected. The effect, therefore, of insufficient\n",
      "investment—investment that failed to keep pace with the steady increase in\n",
      "profits—could be falling total demand reflected in turn in falling orders and\n",
      "output. Again there is no final proof of this point, for unfortunately we do not\n",
      "know how rapidly investment had to grow to keep abreast of the current increase\n",
      "in profits.9 However, the explanation is broadly consistent with the facts.\n",
      "There are other possible explanations of the downturn. Back of the insufficient\n",
      "advance in investment may have been the high interest rates. Perhaps, although\n",
      "less probably, trouble was transmitted to the economy as a whole from some weak\n",
      "sector like agriculture. Further explanations could be offered. But one thing\n",
      "about this experience is clear. Until well along in the autumn of 1929 the\n",
      "downturn was limited. The recession in business activity was modest and\n",
      "underemployment relatively slight. Up to November it was possible to argue that\n",
      "not much of anything had happened. On other occasions, as noted—in 1924 and 1927\n",
      "and of late in 1949—the economy has undergone similar recession. But, unlike\n",
      "these other occasions, in 1929 the recession continued and continued and got\n",
      "violently worse. This is the unique feature of the 1929 experience. This is what\n",
      "we need really to understand.         VThere seems little question that in 1929,\n",
      "modifying a famous cliché, the economy was fundamentally unsound. This is a\n",
      "circumstance of first-rate importance. Many things were wrong, but five\n",
      "weaknesses seem to have had an especially intimate bearing on the ensuing\n",
      "disaster. They are:1) The bad distribution of income. In 1929 the rich were\n",
      "indubitably rich. The figures are not entirely satisfactory, but it seems\n",
      "certain that the 5 per cent of the population with the highest incomes in that\n",
      "year received approximately one third of all personal income. The proportion of\n",
      "personal income received in the form of interest, dividends, and rent—the\n",
      "income, broadly speaking, of the well-to-do—was about twice as great as in the\n",
      "years following the Second World War.10This highly unequal income distribution\n",
      "meant that the economy was dependent on a high level of investment or a high\n",
      "level of luxury consumer spending or both. The rich cannot buy great quantities\n",
      "of bread. If they are to dispose of what they receive it must be on luxuries or\n",
      "by way of investment in new plants and new projects. Both investment and luxury\n",
      "spending are subject, inevitably, to more erratic influences and to wider\n",
      "fluctuations than the bread and rent outlays of the $25-a-week workman. This\n",
      "high-bracket spending and investment was especially susceptible, one may assume,\n",
      "to the crushing news from the stock market in October of 1929. 2) The bad\n",
      "corporate structure. In November 1929, a few weeks after the crash, the Harvard\n",
      "Economic Society gave as a principal reason why a depression need not be feared\n",
      "its reasoned judgment that \"business in most lines has been conducted with\n",
      "prudence and conservatism.\"11 The fact was that American enterprise in the\n",
      "twenties had opened its hospitable arms to an exceptional number of promoters,\n",
      "grafters, swindlers, impostors, and frauds. This, in the long history of such\n",
      "activities, was a kind of flood tide of corporate larceny.The most important\n",
      "corporate weakness was inherent in the vast new structure of holding companies\n",
      "and investment trusts. The holding companies controlled large segments of the\n",
      "utility, railroad, and entertainment business. Here, as with the investment\n",
      "trusts, was the constant danger of devastation by reverse leverage. In\n",
      "particular, dividends from the operating companies paid the interest on the\n",
      "bonds of upstream holding companies. The interruption of the dividends meant\n",
      "default on the bonds, bankruptcy, and the collapse of the structure. Under these\n",
      "circumstances, the temptation to curtail investment in operating plant in order\n",
      "to continue dividends was obviously strong. This added to deflationary\n",
      "pressures. The latter, in turn, curtailed earnings and helped bring down the\n",
      "corporate pyramids. When this happened, even more retrenchment was inevitable.\n",
      "Income was earmarked for debt repayment. Borrowing for new investment became\n",
      "impossible. It would be hard to imagine a corporate system better designed to\n",
      "continue and accentuate a deflationary spiral. 3) The bad banking structure.\n",
      "Since the early thirties, a generation of Americans has been told, sometimes\n",
      "with amusement, sometimes with indignation, often with outrage, of the banking\n",
      "practices of the late twenties. In fact, many of these practices were made\n",
      "ludicrous only by the depression. Loans which would have been perfectly good\n",
      "were made perfectly foolish by the collapse of the borrower's prices or the\n",
      "markets for his goods or the value of the collateral he had posted. The most\n",
      "responsible bankers—those who saw that their debtors were victims of\n",
      "circumstances far beyond their control and sought to help—were often made to\n",
      "look the worst. The bankers yielded, as did others, to the blithe, optimistic,\n",
      "and immoral mood of times but probably not more so. A depression such as that of\n",
      "1929–32, were it to begin as this is written, would also be damaging to many\n",
      "currently impeccable banking reputations.However, although the bankers were not\n",
      "unusually foolish in 1929, the banking structure was inherently weak. The\n",
      "weakness was implicit in the large numbers of independent units. When one bank\n",
      "failed, the assets of others were frozen while depositors elsewhere had a\n",
      "pregnant warning to go and ask for their money. Thus one failure led to other\n",
      "failures, and these spread with a domino effect. Even in the best of times local\n",
      "misfortune or isolated mismanagement could start such a chain reaction. (In the\n",
      "first six months of 1929, 346 banks failed in various parts of the country with\n",
      "aggregate deposits of nearly $115 million.)12 When income, employment, and\n",
      "values fell as the result of a depression bank failures could quickly become\n",
      "epidemic. This happened after 1929. Again it would be hard to imagine a better\n",
      "arrangement for magnifying the effects of fear. The weak destroyed not only the\n",
      "other weak, but weakened the strong. People everywhere, rich and poor, were made\n",
      "aware of the disaster by the persuasive intelligence that their savings had been\n",
      "destroyed. Needless to say, such a banking system, once in the convulsions of\n",
      "failure, had a uniquely repressive effect on the spending of its depositors and\n",
      "the investment of its clients.4) The dubious state of the foreign balance. This\n",
      "is a familiar story. During the First World War, the United States became a\n",
      "creditor on international account. In the decade following, the surplus of\n",
      "exports over imports which once had paid the interest and principal on loans\n",
      "from Europe continued. The high tariffs, which restricted imports and helped to\n",
      "create this surplus of exports remained. However, history and traditional\n",
      "trading habits also accounted for the persistence of the favorable balance, so\n",
      "called.Before, payments on interest and principal had in effect been deducted\n",
      "from the trade balance. Now that the United States was a creditor, they were\n",
      "added to this balance. The latter, it should be said, was not huge. In only one\n",
      "year (1928) did the excess of exports over imports come to as much as a billion\n",
      "dollars; in 1923 and 1926 it was only about $375,000,000.13 However, large or\n",
      "small, this difference had to be covered. Other countries which were buying more\n",
      "than they sold, and had debt payments to make in addition, had somehow to find\n",
      "the means for making up the deficit in their transactions with the United\n",
      "States.During most of the twenties the difference was covered by cash—i.e., gold\n",
      "payments to the United States—and by new private loans by the United States to\n",
      "other countries. Most of the loans were to governments—national, state, or\n",
      "municipal bodies—and a large proportion were to Germany and Central and South\n",
      "America. The underwriters' margins in handling these loans were generous; the\n",
      "public took them up with enthusiasm; competition for the business was keen. If\n",
      "unfortunately corruption and bribery were required as competitive instruments,\n",
      "these were used. In late 1927 Juan Leguia, the son of the President of Peru, was\n",
      "paid $450,000 by J. and W. Seligman and Company and the National City Company\n",
      "(the security affiliate of the National City Bank) for his services in\n",
      "connection with a $50,000,000 loan which these houses marketed for Peru.14\n",
      "Juan's services, according to later testimony, were of a rather negative sort.\n",
      "He was paid for not blocking the deal. The Chase extended President Machado of\n",
      "Cuba, a dictator with a marked predisposition toward murder, a generous personal\n",
      "line of credit which at one time reached $200,000.15 Machado's son-in-law was\n",
      "employed by the Chase. The bank did a large business in Cuban bonds. In\n",
      "contemplating these loans, there was a tendency to pass quickly over anything\n",
      "that might appear to the disadvantage of the creditor. Mr. Victor Schoepperle, a\n",
      "vice-president of the National City Company with the responsibility for Latin\n",
      "American loans, made the following appraisal of Peru as a credit prospect: \n",
      "Peru: Bad debt record, adverse moral and political risk, bad internal debt\n",
      "situation, trade situation about as satisfactory as that of Chile in the past\n",
      "three years. Natural resources more varied. On economic showing Peru should go\n",
      "ahead rapidly in the next 10 years.16  On such showing the National City Company\n",
      "floated a $15,000,000 loan for Peru, followed a few months later by a\n",
      "$50,000,000 loan, and some ten months thereafter by a $25,000,000 issue. (Peru\n",
      "did prove a highly adverse political risk. President Leguia, who negotiated the\n",
      "loans, was thrown violently out of office, and the loans went into default.)In\n",
      "all respects these operations were as much a part of the New Era as Shenandoah\n",
      "and Blue Ridge. They were also just as fragile, and once the illusions of the\n",
      "New Era were dissipated they came as abruptly to an end. This, in turn, forced a\n",
      "fundamental revision in the foreign economic position of the United States.\n",
      "Countries could not cover their adverse trade balance with the United States\n",
      "with increased payments of gold, at least not for long. This meant that they had\n",
      "either to increase their exports to the United States or reduce their imports or\n",
      "default on their past loans. President Hoover and the Congress moved promptly to\n",
      "eliminate the first possibility—that the accounts would be balanced by larger\n",
      "imports—by sharply increasing the tariff. Accordingly, debts, including war\n",
      "debts, went into default and there was a precipitate fall in American exports.\n",
      "The reduction was not vast in relation to total output of the American economy,\n",
      "but it contributed to the general distress and was especially hard on farmers.5)\n",
      "The poor state of economic intelligence. To regard the people of any time as\n",
      "particularly obtuse seems vaguely improper, and it also establishes a precedent\n",
      "which members of this generation might regret. Yet it seems certain that the\n",
      "economists and those who offered economic counsel in the late twenties and early\n",
      "thirties were almost uniquely perverse. In the months and years following the\n",
      "stock market crash, the burden of reputable economic advice was invariably on\n",
      "the side of measures that would make things worse. In November of 1929, Mr.\n",
      "Hoover announced a cut in taxes; in the great no-business conferences that\n",
      "followed he asked business firms to keep up their capital investment and to\n",
      "maintain wages. Both of these measures were on the side of increasing spendable\n",
      "income, though unfortunately they were largely without effect. The tax\n",
      "reductions were negligible except in the higher income brackets; businessmen who\n",
      "promised to maintain investment and wages, in accordance with a well-understood\n",
      "convention, considered the promise binding only for the period within which it\n",
      "was not financially disadvantageous to do so. As a result investment outlays and\n",
      "wages were not reduced until circumstances would in any case have brought their\n",
      "reduction. Still, the effort was in the right direction. Thereafter policy was\n",
      "almost entirely on the side of making things worse. Asked how the government\n",
      "could best advance recovery, the sound and responsible adviser urged that the\n",
      "budget be balanced. Both parties agreed on this. For Republicans the balanced\n",
      "budget was, as ever, high doctrine. But the Democratic Party platform of 1932,\n",
      "with an explicitness which politicians rarely advise, also called for a \"federal\n",
      "budget annually balanced on the basis of accurate executive estimates within\n",
      "revenues...\"A commitment to a balanced budget is always comprehensive. It then\n",
      "meant there could be no increase in government outlays to expand purchasing\n",
      "power and relieve distress. It meant there could be no further tax reduction.\n",
      "But taken literally it meant much more. From 1930 on the budget was far out of\n",
      "balance, and balance, therefore, meant an increase in taxes, a reduction in\n",
      "spending, or both. The Democratic platform in 1932 called for an \"immediate and\n",
      "drastic reduction of governmental expenditures\" to accomplish at least a 25 per\n",
      "cent decrease in the cost of government.The balanced budget was not a subject of\n",
      "thought. Nor was it, as often asserted, precisely a matter of faith. Rather it\n",
      "was a formula. For centuries avoidance of borrowing had protected people from\n",
      "slovenly or reckless public housekeeping. Slovenly or reckless keepers of the\n",
      "public purse had often composed complicated arguments to show why balance of\n",
      "income and outlay was not a mark of virtue. Experience had shown that however\n",
      "convenient this belief might seem in the short nm, discomfort or disaster\n",
      "followed in the long run. Those simple precepts of a simple world did not hold\n",
      "amid the growing complexities of the early thirties. Mass unemployment in\n",
      "particular had altered the rules. Events had played a very bad trick on people,\n",
      "but almost no one tried to think out the problem anew. The balanced budget was\n",
      "not the only strait jacket on policy. There was also the bogey of \"going off\"\n",
      "the gold standard and, most surprisingly, of risking inflation. Until 1932 the\n",
      "United States added formidably to its gold reserves, and instead of inflation\n",
      "the country was experiencing the most violent deflation in the nation's history.\n",
      "Yet every sober adviser saw dangers here, including the danger of runaway price\n",
      "increases. Americans, though in years now well in the past, had shown a penchant\n",
      "for tinkering with the money supply and enjoying the brief but heady joys of a\n",
      "boom in prices. In 1931 or 1932, the danger or even the feasibility of such a\n",
      "boom was nil. The advisers and counselors were not, however, analyzing the\n",
      "danger or even the possibility. They were serving only as the custodians of bad\n",
      "memories.The fear of inflation reinforced the demand for the balanced budget. It\n",
      "also limited efforts to make interest rates low, credit plentiful (or at least\n",
      "redundant) and borrowing as easy as possible under the circumstances.\n",
      "Devaluation of the dollar was, of course, flatly ruled out. This directly\n",
      "violated the gold standard rules. At best, in such depression times, monetary\n",
      "policy is a feeble reed on which to lean. The current economic clichés did not\n",
      "allow even the use of that frail weapon. And again, these attitudes were above\n",
      "party. Though himself singularly open-minded, Roosevelt was careful not to\n",
      "offend or disturb his followers. In a speech in Brooklyn toward the close of the\n",
      "1932 campaign, he said:  The Democratic platform specifically declares, \"We\n",
      "advocate a sound currency to be preserved at all hazards.\" That is plain\n",
      "English. In discussing this platform on July 30, I said, \"Sound money is an\n",
      "international necessity, not a domestic consideration for one nation alone.\" Far\n",
      "up in the Northwest, at Butte, I repeated the pledge ... In Seattle I reaffirmed\n",
      "my attitude...17 The following February, Mr. Hoover set forth his view, as often\n",
      "before, in a famous letter to the President-elect: It would steady the country\n",
      "greatly if there could be prompt assurance that there will be no tampering or\n",
      "inflation of the currency; that the budget will be unquestionably balanced even\n",
      "if further taxation is necessary; that the Government credit will be maintained\n",
      "by refusal to exhaust it in the issue of securities.18 The rejection of both\n",
      "fiscal (tax and expenditure) and monetary policy amounted precisely to a\n",
      "rejection of all affirmative government economic policy. The economic advisers\n",
      "of the day had both the unanimity and the authority to force the leaders of both\n",
      "parties to disavow all the available steps to check deflation and depression. In\n",
      "its own way this was a marked achievement—a triumph of dogma over thought. The\n",
      "consequences were profound.         VIIt is in light of the above weaknesses of\n",
      "the economy that the role of the stock market crash in the great tragedy of the\n",
      "thirties must be seen. The years of self-depreciation by Wall Street to the\n",
      "contrary, the role is one of respectable importance. The collapse in securities\n",
      "values affected in the first instance the wealthy and the well-to-do. But we see\n",
      "that in the world of 1929 this was a vital group. The members disposed of a\n",
      "large proportion of the consumer income; they were the source of a lion's share\n",
      "of personal saving and investment. Anything that struck at the spending or\n",
      "investment by this group would of necessity have broad effects on expenditure\n",
      "and income in the economy at large. Precisely such a blow was struck by the\n",
      "stock market crash. In addition, the crash promptly removed from the economy the\n",
      "support that it had been deriving from the spending of stock market gains.The\n",
      "stock market crash was also an exceptionally effective way of exploiting the\n",
      "weaknesses of the corporate structure. Operating companies at the end of the\n",
      "holding-company chain were forced by the crash to retrench. The subsequent\n",
      "collapse of these systems and also of the investment trusts effectively\n",
      "destroyed both the ability to borrow and the willingness to lend for investment.\n",
      "What have long looked like purely fiduciary effects were, in fact, quickly\n",
      "translated into declining orders and increasing unemployment. The crash was also\n",
      "effective in bringing to an end the foreign lending by which the international\n",
      "accounts had been balanced. Now the accounts had, in the main, to be balanced by\n",
      "reduced exports. This put prompt and heavy pressure on export markets for wheat,\n",
      "cotton, and tobacco. Perhaps the foreign loans had only delayed an adjustment in\n",
      "the balance which had one day to come. The stock market crash served nonetheless\n",
      "to precipitate the adjustment with great suddenness at a most unpropitious time.\n",
      "The instinct of farmers who traced their troubles to the stock market was not\n",
      "totally misguided.Finally, when the misfortune had struck, the attitudes of the\n",
      "time kept anything from being done about it. This, perhaps, was the most\n",
      "disconcerting feature of all. Some people were hungry in 1930 and 1931 and 1932.\n",
      "Others were tortured by the fear that they might go hungry. Yet others suffered\n",
      "the agony of the descent from the honor and respectability that goes with income\n",
      "into poverty. And still others feared that they would be next. Meanwhile\n",
      "everyone suffered from a sense of utter hopelessness. Nothing, it seemed, could\n",
      "be done. And given the ideas which controlled policy, nothing could be done.Had\n",
      "the economy been fundamentally sound in 1929 the effect of the great stock\n",
      "market crash might have been small. Alternatively, the shock to confidence and\n",
      "the loss of spending by those who were caught in the market might soon have worn\n",
      "off. But business in 1929 was not sound; on the contrary it was exceedingly\n",
      "fragile. It was vulnerable to the kind of blow it received from Wall Street.\n",
      "Those who have emphasized this vulnerability are obviously on strong ground. Yet\n",
      "when a greenhouse succumbs to a hailstorm something more than a purely passive\n",
      "role is normally attributed to the storm. One must accord similar significance\n",
      "to the typhoon which blew out of lower Manhattan in October 1929.         VIIThe\n",
      "military historian when he has finished his chronicle is excused. He is not\n",
      "required to consider the chance for a renewal of war with the Indians, the\n",
      "Mexicans, or the Confederacy. Nor will anyone press him to say how such acrimony\n",
      "can be prevented. But economics is taken more seriously. The economic historian,\n",
      "as a result, is invariably asked whether the misfortunes he describes will\n",
      "afflict us again and how they may be prevented.The task of this book, as\n",
      "suggested on an early page, is only to tell what happened in 1929. It is not to\n",
      "tell whether or when the misfortunes of 1929 will recur. One of the pregnant\n",
      "lessons of that year will by now be plain: it is that very specific and personal\n",
      "misfortune awaits those who presume to believe that the future is revealed to\n",
      "them. Yet, without undue risk, it may be possible to gain from our view of this\n",
      "useful year some insights into the future. We can distinguish, in particular,\n",
      "between misfortunes that could happen again and others which events, many of\n",
      "them in the aftermath of 1929, have at least made improbable. And we can perhaps\n",
      "see a little of the form and magnitude of the remaining peril.At first glance\n",
      "the least probable of the misadventures of the late twenties would seem to be\n",
      "another wild boom in the stock market with its inevitable collapse. As those\n",
      "days of disenchantment drew to a close, tens of thousands of Americans shook\n",
      "their heads and muttered, \"Never again.\" In every considerable community there\n",
      "are, even now, a few survivors, aged but still chastened, who are still\n",
      "muttering and still shaking their heads. The New Era had no such guardians of\n",
      "sound pessimism. Also, there are the new government measures and controls. The\n",
      "powers of the Federal Reserve Board—now styled the Board of Governors, the\n",
      "Federal Reserve System—have been strengthened both in relation to the individual\n",
      "Reserve banks and the member banks. Mitchell's defiance of March 1929 is now\n",
      "unthinkable. What was then an act of arrogant but not abnormal individualism\n",
      "would now be regarded as idiotic. The New York Federal Reserve Bank retains a\n",
      "measure of moral authority and autonomy, but not enough to resist any Washington\n",
      "policy. Now also there is power to set margin requirements. If necessary, the\n",
      "speculator can be made to post the full price of the stock he buys. While this\n",
      "may not completely discourage him, it does mean that when the market falls there\n",
      "can be no outsurge of margin calls to force further sales and insure that the\n",
      "liquidation will go through continuing spasms. Finally, the Securities and\n",
      "Exchange Commission is a bar, one hopes effective, to large-scale market\n",
      "manipulation, and it also keeps rein on the devices and the salesmanship by\n",
      "which new speculators are recruited.Yet, in some respects, the chance for\n",
      "recurrence of a speculative orgy remains good. No one can doubt that the\n",
      "American people remain susceptible to the speculative mood—to the conviction\n",
      "that enterprise can be attended by unlimited rewards in which they,\n",
      "individually, were meant to share. A rising market can still bring the reality\n",
      "of riches. This, in turn, can draw more and more people to participate. The\n",
      "government preventatives and controls are ready. In the hands of a determined\n",
      "government their efficacy cannot be doubted. There are, however, a hundred\n",
      "reasons why a government will determine not to use them. In our democracy an\n",
      "election is in the offing even on the day after an election. The avoidance of\n",
      "depression and the prevention of unemployment have become for the politician the\n",
      "most critical of all questions of public policy. Action to break up a boom must\n",
      "always be weighed against the chance that it will cause unemployment at a\n",
      "politically inopportune moment. Booms, it must be noted, are not stopped until\n",
      "after they have started. And after they have started the action will always\n",
      "look, as it did to the frightened men in the Federal Reserve Board in February\n",
      "1929, like a decision in favor of immediate as against ultimate death. As we\n",
      "have seen, the immediate death not only has the disadvantage of being immediate\n",
      "but of identifying the executioner.The market will not go on a speculative\n",
      "rampage without some rationalization. But during any future boom some newly\n",
      "rediscovered virtuosity of the free enterprise system will be cited. It will be\n",
      "pointed out that people are justified in paying the present prices—indeed,\n",
      "almost any price—to have an equity position in the system. Among the first to\n",
      "accept these rationalizations will be some of those responsible for invoking the\n",
      "controls. They will say firmly that controls are not needed. The newspapers,\n",
      "some of them, will agree and speak harshly of those who think action might be in\n",
      "order. They will be called men of little faith.19         VIIIA new adventure in\n",
      "stock market speculation sometime in the future followed by another collapse\n",
      "would not have the same effect on the economy as in 1929. Whether it would show\n",
      "the economy to be fundamentally sound or unsound is something, unfortunately,\n",
      "that will not be wholly evident until after the event. There can be no question,\n",
      "however, that many of the points of extreme weakness exposed in 1929 or soon\n",
      "thereafter have since been substantially strengthened. The distribution of\n",
      "income is no longer quite so lopsided. Between 1929 and 1948 the share of total\n",
      "personal income going to the 5 per cent of the population with the highest\n",
      "income dropped from nearly a third to less than a fifth of the total. Between\n",
      "1929 and 1950 the share of all family income which was received as wages,\n",
      "salaries, pensions, and unemployment compensation increased from approximately\n",
      "61 per cent to approximately 71 per cent. This is the income of everyday people.\n",
      "Although dividends, interest, and rent, the income characteristically of the\n",
      "well-to-do, increased in total amount, the share dropped from just over 22 to\n",
      "just over 12 per cent of total family personal income.20\" In ensuing years the\n",
      "improvement in income distribution tapered off and slightly reversed itself. It\n",
      "remains far better than in the twenties. Similarly after 1929, the great\n",
      "investment trust promotions were folded up and put away, although eventually\n",
      "they were replaced in part, and alas, by mutual and offshore funds, Equity\n",
      "Funding and the Real Estate Investment Trusts, which became the casualties of\n",
      "the post-1970 collapse. However, the SEC, aided by the bankruptcy laws,\n",
      "flattened out the great utility holding company pyramids. Federal insurance of\n",
      "bank deposits, even to this day, has not been given full credit for the\n",
      "revolution that it has worked in the nation's banking structure. With this one\n",
      "piece of legislation the fear which operated so efficiently to transmit weakness\n",
      "was dissolved. As a result one grievous defect of the old system, by which\n",
      "failure begot failure, was cured. Rarely has so much been accomplished by a\n",
      "single law. The problem of the foreign balance is much changed from what it was\n",
      "twenty-five years ago. Now the United States finds itself with a propensity to\n",
      "buy or spend far more than it sells and receives.Finally, there has been a\n",
      "modest accretion of economic knowledge. A developing depression would not now be\n",
      "met with a fixed determination to make it worse. Without question, ceremonial\n",
      "conferences would be assembled at the White House. We would see an explosion of\n",
      "reassurance and incantation. Many would urge waiting and hoping as the best\n",
      "policy. Not again, however, would people suppose that the best policy would\n",
      "be—as Secretary Mellon so infelicitously phrased it—to \"liquidate labor,\n",
      "liquidate stocks, liquidate the farmers, liquidate real estate.\"21 Our\n",
      "determination to deal firmly and adequately with a serious depression is still\n",
      "to be tested. But there is still a considerable difference between a failure to\n",
      "do enough that is right and a determination to do much that is wrong.Other\n",
      "weaknesses in the economy have been corrected. The much maligned farm program\n",
      "provides a measure of security for farm income and therewith for spending by\n",
      "farmers. Unemployment compensation accomplishes the same result, if still\n",
      "inadequately, for labor. The remainder of the social security system—pensions\n",
      "and public assistance—helps protect the income and consequently the expenditures\n",
      "of yet other segments of the population. The tax system is a far better servant\n",
      "of stability than it was in 1929. An angry god may have endowed capitalism with\n",
      "inherent contradictions. But at least as an afterthought he was kind enough to\n",
      "make social reform surprisingly consistent with improved operation of the\n",
      "system.         IXYet all this reinforcement notwithstanding, it would be unwise\n",
      "to expose the economy to the shock of another major speculative collapse. Some\n",
      "of the new reinforcements might buckle. Fissures might open at other new and\n",
      "perhaps unexpected places. Even the quick withdrawal from the economy of the\n",
      "spending that comes from stock market gains might be damaging. Any collapse,\n",
      "even though the further consequences were small, would not be good for the\n",
      "public reputation of Wall Street.Wall Street, in recent times, has become, as a\n",
      "learned phrase has it, very \"public relations conscious.\" Since a speculative\n",
      "collapse can only follow a speculative boom, one might expect that Wall Street\n",
      "would lay a heavy hand on any resurgence of speculation. The Federal Reserve\n",
      "would be asked by bankers and brokers to lift margins to the limit; it would be\n",
      "warned to enforce the requirement sternly against those who might try to borrow\n",
      "on their own stocks and bonds in order to buy more of them. The public would be\n",
      "warned sharply and often of the risks inherent in buying stocks for the rise.\n",
      "Those who persisted, nonetheless, would have no one to blame but themselves. The\n",
      "position of the Stock Exchange, its members, the banks, and the financial\n",
      "community in general would be perfectly clear and as well protected in the event\n",
      "of a further collapse as sound public relations allow.As noted, all this might\n",
      "logically be expected. However, it did not happen in the go-go years of the late\n",
      "sixties and immediately after—the years of the performance funds and the\n",
      "conglomerate explosion—nor will it come to pass. This is not because the\n",
      "instinct for self-preservation in Wall Street is poorly developed. On the\n",
      "contrary, it is probably normal and may be above. But now, as throughout\n",
      "history, financial capacity and political perspicacity are inversely correlated.\n",
      "Long-run salvation by men of business has never been highly regarded if it means\n",
      "disturbance of orderly life and convenience in the present. So inaction will be\n",
      "advocated in the present even though it means deep trouble in the future. Here,\n",
      "at least equally with communism, lies the threat to capitalism. It is what\n",
      "causes men who know that things are going quite wrong to say that things are\n",
      "fundamentally sound.          Index Acceptances, [>], [>]Agriculture. [>]; farm\n",
      "program, [>]; and stock market, [>]; weakness of economy, [>]. See also\n",
      "FarmersAktiebolaget Kreuger and Toll. See Kreuger and TollAldrich, Winthrop W.,\n",
      "[>]–[>]All Quiet on the Western Front, [>]Alleghany Corporation, [>], [>]Allen,\n",
      "Frederick Lewis, [>], [>] (n)., [>] (n)., [>] (n)., [>] (n)., [>], [>], [>],\n",
      "[>], [>] (n)., [>] (n)., [>] (n).Allied Chemical and Dye, [>]American Can, [>],\n",
      "[>]American Company, [>]American Founders Group, [>], [>]–[>], [>]American\n",
      "Insuranstocks Corporation, [>]The American Magazine, [>]American Railway\n",
      "Express, [>]American Stores, [>]American Telephone and Telegraph Company, [>],\n",
      "[>], [>]–[>], [>], [>], [>], [>]–[>]American Tobacco, [>]American Trust Company,\n",
      "[>]Anaconda Copper, [>]Anglo-American Shares, Inc., [>]Angly, Edward, [>] (n).,\n",
      "[>] (n).Aquinas, Saint Thomas, [>]Armour and Company, [>]Arndt, H. W., [>]\n",
      "(n).Associated Cas and Electric, [>], [>]Astor, Vincent, [>]The Atlantic\n",
      "Monthly, [>]Auburn Automobile, [>]Axley, Seth, [>]Ayres, Colonel Leonard P.,\n",
      "[>]Babe Ruth, [>]Babson, Roger W., [>], [>], [>]; Annual National Business\n",
      "Conference, [>] Babson break, [>]–[>]; election forecast, [>]Bagehot, Walter,\n",
      "[>], [>] (n), [>], [>]Baker, George F., Jr., [>] (n).Balance of payments. See\n",
      "Foreign balanceBalanced budget, a formula, [>]–[>]; Whitney and, [>]Baldwin\n",
      "Locomotive Works, [>]Bank of England, [>], [>]Bank of France, [>]Bankers, bear\n",
      "raid denied, [>]–[>]; loss of prestige, [>]–[>], [>], [>]; optimism of, [>];\n",
      "Pecora committee, [>]; speculation of officers in their own stocks, [>], [>],\n",
      "[>]; support market, [>]–[>], [>]–[>]; support ended, [>]–[>], [>]Bankers Trust\n",
      "Company, [>]Banks, failures, [>]; federal insurance of deposits, [>]; in another\n",
      "crisis, [>]; out-of-town banks recall money from Wall Street, [>]; weakness of\n",
      "banking structure, [>]–[>]. See also Bankers, Commercial banks, Federal Reserve\n",
      "System, New York banksBarron's, [>], [>] (n)., [>], [>], [>]Barton, Bruce,\n",
      "[>]Baruch, Bernard, [>]Bennett, John J., [>]Berlin, Irving, [>]Bethlehem Steel,\n",
      "[>]Biddle, Nicholas, [>]Black Thursday. See New York Stock Exchange Blue Ridge\n",
      "Corporation, [>], [>], [>], [>], [>], [>], [>], [>]Blyth and Company,\n",
      "[>]Bonbright and Company, [>]Boston, [>], [>], [>], [>], [>]Boston Association\n",
      "of Stock Exchange Firms, [>] (n).Boston Edison, [>]Boston Globe, [>]Brisbane,\n",
      "Arthur, [>], [>], [>]Britain, general strike, [>]; gold standard, [>]–[>],\n",
      "Restoration, [>]Brokers. [>], [>], [>], [>], [>]Brokers loans, [>], [>], [>],\n",
      "[>], [>], [>]; explanation of, [>]; as index of volume of speculation, [>]–[>],\n",
      "[>], [>]–[>], [>]–[>], [>]–[>], [>], [>], [>]; investment trusts, [>]; largest\n",
      "weekly drop, [>]; warnings concerning, [>]Brookhart, Senator Smith W., [>],\n",
      "[>]–[>]Brooklyn-Manhattan Trust Company, [>]Brownell, Herbert, [>]Bryan, William\n",
      "Jennings, [>]Bull market, [>], [>], [>], [>], [>]. See also Stock marketBusiness\n",
      "cycle: incantation and, [>]; law of compensation, [>], [>]–[>]; and speculation,\n",
      "[>]Cabot, Paul C., [>]Caesar, Julius, [>]Call market. See Brokers' loansCanada,\n",
      "Foshay, [>]; Wiggin, [>]Cantor Eddie, [>]Capitalism, [>]Carisbrooke, Marquess\n",
      "of, [>]Carton, Sydney, [>]Case, J. I., [>], [>], [>], [>], [>]Catchings,\n",
      "Waddill, [>]Central America, Foshay, [>]; loans to, [>]Central banking. See\n",
      "Federal Reserve SystemCentral States Electric Corporation, [>], [>]Chain stores,\n",
      "[>]Chase National Bank, [>], [>], [>], [>]; and Wiggin, [>]–[>]Chicago, Board of\n",
      "Trade, [>]; gangsters, [>]–[>]; stock exchange, [>], [>]Chile, [>]Churchill,\n",
      "Winston, [>], [>]Cincinnati, [>]Cities Service, [>]Clovis, New Mexico,\n",
      "[>]Columbus, Ohio, [>]The Commercial Financial Chronicle, [>] (n) [>], [>] (n).,\n",
      "[>], [>]Commercial banks, and Federal Reserve, [>]–[>], [>]–[>]; investment\n",
      "trusts, [>]; loans, [>]–[>], [>]; and Mitchell, [>]; and securities affiliates,\n",
      "[>]–[>], [>]Commercial National Bank and Trust Company, [>]Commercial Solvents,\n",
      "[>]Commodity markets, [>]Common stocks, [>], [>]–[>], [>], [>]–[>], [>], [>],\n",
      "[>]–[>], [>], [>], [>], [>]–[>]Commonwealth and Southern, [>]Communists, [>],\n",
      "[>], [>]Congress, [>]–[>], [>], [>]; committees, [>]; investigations, [>],\n",
      "[>]Consumer spending, [>], [>], [>], [>], [>]Continental Illinois Bank,\n",
      "[>]Coolidge, Calvin, [>]–[>], [>], [>], [>], [>] [>]Coral Gables, [>]Com\n",
      "Exchange Bank, [>]Corporate structure, [>]–[>], [>]Corporations, flow of funds\n",
      "to market, loans in the market, [>]–[>]; recall funds, [>]Corsair,\n",
      "[>]Cosmopolitan Fiscal Corporation, [>]Cotton, [>]County Trust Company,\n",
      "[>]Couzens, Senator James, [>]–[>]Credit : easy credit not cause of speculation,\n",
      "[>]; easy money policy, [>], [>], [>]; plentiful, [>]Crissinger, Daniel R.,\n",
      "[>]Cummings, Attorney General Homer, [>]Curb Exchange, [>], [>], [>], [>],\n",
      "[>]Cutten, Arthur W., [>], [>], [>], [>], [>]–[>]Cyrus, [>] Daily News (New\n",
      "York), [>]Davis, Howland S., [>]Davis, Joseph S., [>]Day, Edmund E.,\n",
      "[>]Deflation, [>], [>], [>]Democratic National Committee, [>], [>]Democrats,\n",
      "[>], [>], [>]; and balanced budget, [>]; sound currency, [>]Department store\n",
      "sales, [>]–[>]Depression [>], [>], [>], [>], [>], [>], [>]; Babson, [>]; Harvard\n",
      "Economic Society, [>]–[>]; and market collapse, [>]; of 1920–[>], [>], [>], [>],\n",
      "[>]. See also Business cycleDewey, Thomas E., [>], [>]–[>]Dice, Professor\n",
      "Charles Amos, [>], [>] [>] (n)., [>], [>]–[>], [>]Distilled Liquors Corporation,\n",
      "[>]–[>]Dividends [>], [>], [>]Dodsworth, [>]Dollar, devaluation of,\n",
      "[>]–[>]Douglas, William O., [>]Dow-Jones averages, [>] (n)., [>]Dulles, John\n",
      "Foster, [>]Du Pont (E. I. du Pont de Nemours and Company), [>]Du Ponts, [>].\n",
      "[>]Durant, Donald, [>], [>] (n).Durant, William Crapo, [>], [>], [>], [>]Eastman\n",
      "Kodak, [>]Economic history: insights, [>]–[>]; lessons of, [>]Economic policy,\n",
      "added economic knowledge, [>]; inadequacy of, [>]–[>], [>]; and politics, [>]The\n",
      "Economist, [>]Eisenhower, President Dwight D., [>]Election 1928, forecasts, [>];\n",
      "Hoover victory and boom, [>]Electric Bond and Share Company, [>]Embezzlement,\n",
      "boom and crash, [>]–[>]; Union Industrial Bank, [>]Employment, [>], [>], [>],\n",
      "[>], [>]Equitable Trust Company, [>]–[>]Erleigh, Viscount, [>] (n).Export-Import\n",
      "Bank, [>]Exports. See Foreign balanceFarley, James A., [>]Farmers, [>], [>]. See\n",
      "also Agriculture.Federal Reserve Bank of Minneapolis, [>]Federal Reserve Bank of\n",
      "New York, [>], [>]–[>], [>]–[>], [>]; rediscount rate, [>], [>], [>]–[>],\n",
      "[>]Federal Reserve Board. See Federal Reserve System, BoardFederal Reserve\n",
      "Bulletin, [>] (n).Federal Reserve index of factory production, [>], [>]Federal\n",
      "Reserve index of industrial production, [>], [>], [>], [>], 1974Federal Reserve\n",
      "System; Board of Governors, [>]; in another crisis, [>]; loans to New York\n",
      "banks, [>], [>]; margin requirements, [>], [>], [>], [>]; and Miller, [>];\n",
      "mystique of central banking, [>]; new powers of, [>]; open market operations,\n",
      "[>]–[>], [>]; rediscount rate, [>], [>]–[>], [>]; and Warburg, [>]; weekly\n",
      "record of brokers' loans, [>], willful helplessness, [>]Federal Reserve System,\n",
      "Board; Blame for speculation, [>]; 1920–[>] depression, [>]; Coolidge, [>];\n",
      "criticism of, [>]–[>]; easy money policy, [>]; hands-off policy, [>]; Hoover,\n",
      "[>]; March meetings, [>]–[>]; Mitchell, [>]–[>]; moral suasion, [>]–[>], [>];\n",
      "primary role and incompetence, [>]–[>], [>] rediscount rate, [>], [>]Fess,\n",
      "Simeon D., [>]Financial and Industrial Securities Corporation, [>]Financial\n",
      "Counselor, [>]Financial Panic of 1907, [>], [>]First National Bank of New York,\n",
      "[>] (n).Fisher brothers, [>]Fisher, Professor Irving, [>], [>], [>]–[>], [>],\n",
      "[>], [>], [>] Florida, hurricanes, [>]–[>], [>]; land boom, [>]–[>], [>], [>],\n",
      "[>]; trading in \"binders,\" [>]–[>]; Whitney, [>]Foch, Marshal, [>]Ford, Henry,\n",
      "[>], [>]Ford Motor Company, reduced prices, [>]; shutdown, [>]Forecasts, [>],\n",
      "[>], [>]–[>], [>], [>], [>], [>]–[>], [>]–[>]Foreign balance, [>]–[>], [>],\n",
      "[>]Foreign loans. See Foreign balanceFoshay, failure of, [>]–[>]Foster, William\n",
      "Z., [>]Freight-car loadings, [>], [>], [>], [>]Friday, Dr. David, [>]Garrett,\n",
      "Garet, [>] (n).Gay, Charles R., [>], [>]General Electric Company, [>], [>], [>],\n",
      "[>], [>], [>], [>], [>]General Motors Corporation, [>]–[>], [>], [>], [>]; Sloan\n",
      "president of, [>]Germany, [>], [>], [>]Gifford, Walter, [>]Glass, Senator\n",
      "Carter, [>], [>], [>]Gold and Babson, [>]; flow to New York, [>]Gold standard,\n",
      "[>], [>], [>]–[>]; Britain and, [>]Goldman, Sachs and Company, [>] (n), [>],\n",
      "[>], [>], [>], [>], [>]Goldman Sachs Trading Corporation, [>], [>] [>], [>],\n",
      "[>]Goldsmith, Selma, [>] (n).Good, James W., [>]Government regulation, [>],\n",
      "[>]–[>], [>]–[>], [>]–[>]; SEC, [>]–[>]Graf Zeppelin, [>]Great Crash,\n",
      "recurrence, [>]–[>]; relationship to the depression, [>], [>]; special\n",
      "characteristic of 1929, [>], [>]–[>]Great Depression causes of, [>]–[>];\n",
      "relation to stock market crash, [>], [>]–[>]; ten years of, [>]Gresham's Law,\n",
      "[>]Gross National Product, [>]Guaranty Trust Company, [>]Gude Winmill Trading\n",
      "Corporation, [>]Hague, Mayor Frank, [>], [>]Hanes, John W., [>]Harding, Warren\n",
      "G., [>]Harpers Magazine, [>] (n).Harris, Seymour E., [>] (n).Harrison, George\n",
      "L., [>]Harvard Economic Society, [>], [>], [>]–[>], [>], [>]Hatry, Clarence,\n",
      "[>]–[>], [>]Hearst papers, Brisbane in, [>]Hiss Alger, [>]–[>]Holding company,\n",
      "[>], [>]; bank holding companies, [>]; bad corporate structure, [>], [>]Home-\n",
      "building industry, [>]Hoover, Herbert, [>], [>], [>] (n)., [>], [>], [>],\n",
      "[>]–[>], [>], [>], [>]–[>], [>], [>], [>]; attempts at reassurance, [>]–[>];\n",
      "balanced budget, [>]; concern over speculation, [>], [>]; no-business meetings,\n",
      "[>]–[>], [>], [>]; tax cut, [>]Hopson, Howard C., [>], [>], [>], [>]Hornblower\n",
      "and Weeks, [>], [>]–[>]Hugh-Jones, E. M., [>] (n).Incantation, [>], [>], [>],\n",
      "[>]. See also Business cycleIncome, [>], [>], [>], [>]; distribution of, [>],\n",
      "[>], [>]; farm income, [>]; state of economy in 1929, [>] [>]Incorporated\n",
      "Investors, [>]Inflation, [>]Insult, Samuel, [>], [>], [>], [>], [>]Insull\n",
      "Utility Investments, Inc., [>], [>]Interest rate, [>]; Federal Reserve, [>];\n",
      "high interest rate, [>]; in margin trading, [>]; National City and, [>]; no\n",
      "check to speculator, [>], [>]International Acceptance Bank, [>]International\n",
      "Bank for Reconstruction and Development, [>]International Carriers, Ltd.,\n",
      "[>]International Harvester, [>]International Nickel, [>]Inventory recession,\n",
      "[>]–[>]Investment, [>], 1S6; failure to keep pace with profits, [>]; high\n",
      "bracket, [>], [>] Investment banks, sponsors of investment trusts,\n",
      "[>]–[>]Investment company. See Investment trustsInvestment corporation. See\n",
      "Investment trustsInvestment News, [>] (n)., [>] (n).Investment trusts, [>],\n",
      "[>]–[>], [>], [>], [>]; development and nature of. [>]–[>]; expansion, [>]–[>];\n",
      "and genius, [>]–[>]; July 1932, [>]; Lehman Corporation, [>]; leverage, [>]–[>],\n",
      "[>], [>], [>]–[>]; Raskob's plan, [>]–[>]; securities of, [>]–[>]; weaknesses\n",
      "of, [>], [>]. See also Goldman, Sachs and Company; United Founders\n",
      "CorporationInvestment Trusts and Investment Companies. See Securities and\n",
      "Exchange CommissionIron Age, [>]Ivy League, [>]–[>].Jacksonville, [>]Jaszi,\n",
      "George, [>] (n).Jay Cooke and Company, [>]Jefferson, Thomas, [>]Joint Committee\n",
      "on the Economic Report, [>] (n).The Journal of Land and Public Utility\n",
      "Economics, [>] (n).Journalists, skepticism toward boom, [>]; in pay of\n",
      "speculators, [>]Kaitz, Hyman, [>] (n).Kemmerer, Professor Edwin W., [>]Keynes,\n",
      "John Maynard, [>], [>] (n).Kingdom of Yugoslavia, [>]Klein, Dr. Julius,\n",
      "[>]Knight, Peter O., [>]Kolo Products Corporation, [>]Kreuger, Ivar, [>], [>],\n",
      "[>], [>], [>], [>]Kreuger and Toll, [>]–[>], [>], [>]Ladies' Home Journal, [>]La\n",
      "Guardia, F. H., [>]Lamont, Robert P., Secretary of Commerce, [>]Lamont, Thomas\n",
      "W., [>], [>], [>]–[>], [>]Latin America, [>]. See also Central and South\n",
      "AmericaLawes, Warden of Sing Sing, [>]Lawrence, Joseph Stagg, [>]–[>], [>], [>],\n",
      "[>]Lee, Higginson and Company, [>], [>] (n).Lef£vre, Edwin, [>], [>] (n).Leguia,\n",
      "President of Peru, [>]Leguia, Juan, [>]Lehman Corporation, [>]Lehman, Herbert,\n",
      "[>]Lenin, [>]Levenson, John J., [>]Leverage. See Investment trustsLiberty bonds,\n",
      "[>]Liebenberg, Maurice, [>] (n).Lindbergh, Charles A., [>]Lion, David M., [>]The\n",
      "Literary Digest, [>] (n)., [>], [>] (n)., [>], [>] (n).Livermore, Jesse L., [>],\n",
      "[>]London, penny press, [>]Macaulay, Thomas Babington, [>]Machado, President of\n",
      "Cuba, [>]Magazine of Wall Street, [>] (n).Manhattan Estates, [>]Manufacturing\n",
      "establishments, number and value of output, [>]Marcosson, Isaac F., [>]Margin\n",
      "trading, [>]–[>], [>]; amount of, [>]; brokers' loans, [>]–[>]; explanation and\n",
      "uses of, [>]–[>]; and Federal Reserve, [>], [>], [>], [>]; margin calls,\n",
      "[>]–[>], [>]; as speculative index, [>]. See also SpeculatorsMarion, Ohio,\n",
      "[>]Mark Twain, [>], [>]Marne, Battle of, [>]Massachusetts Department of Public\n",
      "Utilities, [>]–[>], [>]Master of Sempill, AFC, Colonel, [>]–[>]McAdoo, William\n",
      "Gibbs, [>]McMahon Institute of Economic Research, [>]McMahon, William J.,\n",
      "[>]McNeel's Financial Service, [>] Meehan, M. J., [>], [>], [>]–[>]Mellon,\n",
      "Andrew W., [>]–[>], [>], [>], [>], [>], [>]Mergers, [>]–[>], [>], [>]; National\n",
      "City and Com Exchange Bank, [>]Mexico, Foshay, [>]Miami, [>], [>], [>]Miami\n",
      "Beach. [>]Midland Bank of Cleveland, [>]Miller, Adolph C., [>], [>]Minneapolis,\n",
      "[>], [>]–[>]Mitchell, Charles E., [>], [>]–[>], [>], [>], [>], [>], [>]; arrest\n",
      "[>]; and brokers' loans, [>]; Class A director, New York, Federal Reserve, [>];\n",
      "defiance of Federal Reserve, [>]–[>], [>]; Morgan loan, [>]–[>]; resignation,\n",
      "[>]; rumors of resignation, [>]; salary, [>]–[>]; tax liabilities, [>]–[>];\n",
      "trial, [>]–[>]Mitchell, Wesley Clair, [>] (n).Montgomery Ward, [>], [>], [>],\n",
      "[>], [>], [>]Moore, Geoffrey H., [>] (n).Morgan, J. P., [>], [>]Morgan, J. P.,\n",
      "the elder, [>], [>]Morgan, J. P., and Company. [>]–[>], [>]–[>], [>]–[>];\n",
      "Mitchells loan, [>]–[>]; and Whitney, [>], [>]Mount Pelee, [>]Murlyn\n",
      "Corporation, [>]Mussolini, Benito, [>]Myers, William Starr, [>] (n).National\n",
      "Bureau of Economic Research, [>]National City Bank, [>], [>]–[>], [>], [>], [>],\n",
      "[>], [>]–[>], [>], [>]; merger with Corn Exchange, [>]–[>]; salaries of\n",
      "officers, [>]–[>]; stock of, [>]–[>]National City Company, [>]National Republic\n",
      "Investment Trust, [>]National Waterworks Corporation, [>]Nettie, city of, [>]New\n",
      "Deal, [>], [>], [>], [>]New York banks, loans for margin trading, [>]–[>]; loans\n",
      "from Federal Reserve, [>], [>]; loans increased during crisis, [>]–[>]. See also\n",
      "Banks; Chase National Bank; National City BankNew York Central, [>]New York\n",
      "Herald Tribune, [>] (n).New York Stock Exchange, [>], [>], [>], [>], [>]–[>],\n",
      "[>], [>], [>]; Committee on Business Conduct, [>]; Committee on the Stock List,\n",
      "[>]Days of panic: Black Thursday, [>]–[>], [>], [>]; Monday. October [>], [>];\n",
      "Tuesday, October [>], [>]–[>], [>]Governing Committee, [>], [>]; impersonal\n",
      "market, [>]–[>]; investigating Radio, [>]; and short selling, [>]; investigation\n",
      "of, [>]–[>]; investment trusts, [>]; Kreuger's death, news withheld, [>];\n",
      "manipulation, [>]; and member firms, [>]; organization of, [>]; and other\n",
      "exchanges, [>]–[>]; question of closing, [>]–[>]; regulation of, (SEC), [>];\n",
      "short sessions, [>], [>]–[>]; symbol of evil, [>], [>]; trading volume, [>],\n",
      "[>], [>]–[>], [>], [>], [>]–[>], [>], [>]–[>], [>]–[>] [>], [>]–[>], [>]–[>],\n",
      "[>], [>], [>]. [>]; transoceanic brokerage, [>]–[>]. See also Wall Street, Stock\n",
      "marketNew York Stock Exchange Year Book, [>] (n)– [>] (n)., [>] (n).; tabulation\n",
      "of brokers' loans, [>]New York Times, [>], [>], [>], [>] (n)., [>] (n)., [>],\n",
      "[>], [>], [>], [>], [>] (n)., [>], [>], [>], [>], [>]New York Times industrial\n",
      "averages, [>], [>] (n). (in 1924); [>], [>] (1925–27); [>], [>] (in 1928); [>],\n",
      "[>], [>], [>], [>]–[>], [>], [>], [>], [>], [>], [>], [>], [>], [>] (in 1929),\n",
      "[>] (in 1932)New York World, [>] (n).Newton, Walter H., [>] (n).No-business\n",
      "meetings, [>]–[>], [>], [>]Norman, Montagu, [>], [>], [>]The North American\n",
      "Review, [>]Noyes, Alexander Dana, [>] (n)., [>], [>] (n). Open market\n",
      "operations. See Federal Reserve SystemOptions, [>]Organized support. See Stock\n",
      "MarketOtis Elevator, [>]Pacific American Associates, [>]Palm Beach,\n",
      "[>]Paramount-Famous-Lasky, [>]Paris, police, [>]Parker, Chauncey D.,\n",
      "[>]–[>]Payne, Will, [>]Pecora Committee, [>] (n)., [>]. See also Senate\n",
      "Committee on Banking and CurrencyPecora, Ferdinand, [>]Pennroad, [>]Peru,\n",
      "[>]–[>]Philadelphia Record, [>]Pocantico Hills, [>]Ponzi, Charles, [>]–[>],\n",
      "[>]Pools, [>], [>]–[>], [>], [>]–[>], [>]Poor's Weekly Business and Investment\n",
      "Letter, [>]Potter, William C., [>]Prices, Fisher on stock prices, [>]; in the\n",
      "twenties, [>], [>], [>]; October [>], [>]–[>]; prices allowed to fall, [>];\n",
      "stocks dumped on market, [>]; support for stock prices, [>], [>]Princeton\n",
      "University Press, [>]Production, [>], [>], [>]–[>]; automobiles, [>], [>]; coal,\n",
      "[>]; during Depression, [>]; factory production, [>]; industrial production,\n",
      "[>], [>], [>]; pig iron, [>], [>]; steel, [>], [>], [>], [>]Prohibition, [>],\n",
      "[>]–[>], [>]Prosperity, [>]–[>], [>], [>], [>]; and speculation, [>]. See also\n",
      "Business cycleProsser, Seward, [>]Proust, [>]Radice, E. A., [>] (n).Radio (\n",
      "Radio Corporation of America), [>], [>], [>], [>], [>], [>], [>], [>]Raskob,\n",
      "John J., [>]–[>], [>]–[>], [>], [>], [>]Rediscount rate. See Federal Reserve\n",
      "SystemReis, Bernard J., [>] (n)., [>] (n).Reischsbank, [>]Republican National\n",
      "Committee, [>]Republicans, [>], [>], [>]; balanced budget, [>], [>]; identified\n",
      "with Wall Street, [>]; and Whitney, [>]Rhodes Cecil, [>]Richard Whitney and\n",
      "Company. See Whitney, RichardRiordan, J. J., [>]–[>]Rist Charles, [>]Robbins,\n",
      "Professor Lionel, [>], [>] (n)., [>] (n).Robinson, Henry M., [>]Rochester Gas\n",
      "and Electric Company, [>]Rockefeller, John D., pegging Standard Oil, [>]–[>],\n",
      "[>]; purchasing sound common stocks, [>], [>]Rockefeller, Percy A., [>],\n",
      "[>]Rockefellers, [>]Roosevelt, Franklin D., [>]–[>], [>], [>], [>]Rosenwald,\n",
      "Julius, [>]Royal Aeronautical Society, London, [>]Russia, [>], [>], [>]Sachs,\n",
      "Walter E., [>]–[>]Sales against the box, [>]San Francisco, [>]Santa Fe,\n",
      "[>]Saturday Evening Post, [>], [>] (n)., [>] (n).Savings, [>]; destroyed, [>];\n",
      "and speculation, [>]–[>]Schacht, Hjalmar, [>], [>]Schoepperle, Victor,\n",
      "[>]Schwab, Charles M., [>]Seaboard Air Line, [>], [>]–[>]Seaboard Utilities\n",
      "Shares Corporation, [>]Securities Act of 1933, [>]Securities Exchange Act 1934,\n",
      "[>], [>]Securities and Exchange Commission (SEC), [>], [>], [>], [>],\n",
      "[>]Investment Trusts and Investment Companies, Report, [>] (n)., [>] (n)., [>]\n",
      "(n)., [>] (n)., [>] (n)., [>] (n)., [>] (n)., [>] (n)., [>] (n). Securities and\n",
      "Exchange Commission in the Matter of Richard Whitney, et al., [>] (n). [>] (n).,\n",
      "[>] (n).Securities market. See Stock marketSeligman and Company, J. and W.,\n",
      "[>]Senate, [>]; senators' salaries, [>]–[>]Senate Committee on Banking and\n",
      "Currency, [>]Stock Exchange Practices, Hearings: [>] (n)., [>] (n)., [>] (n).,\n",
      "[>] (n)., [>] (n)., [>]., [>] (n)., [>] (n)., [>] (n)., [>] (n)., [>]\n",
      "(n).Report: [>] (n)., [>] (n)., [>] (n)., [>] (n)., [>] (n)., [>] (n)., [>]\n",
      "(n)., [>] (n)., [>] (n)., [>] (n) [>] (n).Shenandoah Corporation, [>], [>], [>],\n",
      "[>], [>], [>], [>]Shermar Corporation, [>]–[>]Short selling, [>]–[>], [>],\n",
      "[>]Simmons Company, [>]Simmons, Edward H. H., [>] (n)., [>], [>]Sinclair\n",
      "Consolidated Oil Company, [>], [>]Sinclair, Harry F., [>], [>], [>]Sloan, Alfred\n",
      "P., Jr., [>], [>]Smith, Alfred E., [>], [>], [>]–[>]Smith, Bernard E., [>],\n",
      "[>]Social Security, [>]Socialists, [>], [>]Solvay American Investment\n",
      "Corporation, [>]South America, loans, [>]South Sea Bubble, [>], [>], [>], [>],\n",
      "[>]Sparling, Earl, [>] (n).Speculation, [>], [>]–[>], [>]; boom begins in\n",
      "earnest, [>]–[>]; brokers' loans index of, [>]–[>]; characteristics of\n",
      "speculative periods, [>], [>]–[>], [>]–[>]. [>], [>]. [>], [>]–[>], [>]–[>],\n",
      "collapse of, [>]; control of, [>]–[>] [>]; and Coolidge, [>]; devices of,\n",
      "[>]–[>]; estimated participation in, [>]; exposure of, [>]; and Federal Reserve,\n",
      "[>]–[>], [>]; Florida boom, [>]–[>], [>]–[>], [>]; and Hoover, [>]; immunizing\n",
      "effect of collapse, [>], [>]; industrial stocks focus of, [>] (n).; reasons for\n",
      "orgy, [>]–[>]; responsibility for crash, [>]; safeguards against [>]–[>]; and\n",
      "Warburg, [>]. See also Margin trading, Speculators, Stock MarketSpeculators,\n",
      "[>]–[>], [>], [>]–[>], [>], [>], [>], [>], [>], [>]. [>]–[>], [>], [>]; causing\n",
      "crash, [>]–[>]; estimated number of, [>]–[>]; investigation of famous operators,\n",
      "[>]; margin calls, [>]–[>], [>], [>], [>], [>], [>], [>]; panic, [>]; suicides,\n",
      "[>], [>]–[>]Spokesman-Review, Spokane, [>]Standard Oil of New Jersey, [>], [>],\n",
      "[>]–[>], [>]Standard Statistics Company, [>]State Street Investment Corporation,\n",
      "[>]Stevens, Eugene M., [>]Stock exchanges, Boston, [>]; Buffalo, [>]; Chicago,\n",
      "[>], [>]; out-of-town exchanges, [>], [>]; panic, [>]; regulation of (SEC),\n",
      "[>]Stock exchange firms, [>]. [>]; exhaustion, [>], [>]; failures. [>], [>],\n",
      "[>]; moral standards of, [>]; sponsors of investment trusts, [>]; transoceanic\n",
      "brokerage, [>]–[>]. See also BrokersStock Exchange Practices. See Senate\n",
      "Committee on Banking and CurrencyStock MarketBabson break, [>] big operators,\n",
      "[>]–[>], [>]–[>]; Black Thursday, [>]–[>], [>]Boom: survived Florida, [>];\n",
      "rising prices in twenties, [>]–[>]; March 1928, [>]–[>]; June 1928, [>];\n",
      "election forecasts, [>]; post-election boom, [>]–[>]; February 1929 setback,\n",
      "[>]; March nervousness, [>]–[>]; summer spurt, [>]Bull market ended, [>]; center\n",
      "of immorality, [>]–[>], [>]; impersonal market, [>], [>]; 1930–[>], [>]–[>];\n",
      "October prelude 1929, [>]–[>]; official optimism, sober predictions, [>]–[>];\n",
      "organized support, [>]–[>], [>], [>]–[>], [>], [>], [>], [>]; panic, [>]–[>];\n",
      "predominance of, [>]–[>]; reassurance, [>], [>], [>] (see also Incanta tion) as\n",
      "reflection of economic situation, [>]; search for wrongdoers, [>], [>] [>];\n",
      "slumps, [>]–[>], [>]. See also Margin trading, Speculators, Wall Street The\n",
      "Stock Market Crash—and After, [>]Strong, Governor Benjamin, [>]–[>]Suicides,\n",
      "Kreuger, [>]; number of, [>]–[>]; rate, [>]–[>]; Riordan. [>]–[>]Sullivan,\n",
      "Lawrence, [>] (n).Syndicates, [>], [>], [>]–[>]Tammany Hall, [>]Tariffs, [>],\n",
      "[>]Taxes, [>], [>], [>], [>]; avoidance of, [>]; and balanced budget; [>], [>];\n",
      "Mitchell, [>]–[>]Teagle, Walter, [>], [>]Teapot Dome, [>]Television, [>]Time\n",
      "Magazine, [>]\"The Trader\" [>]Trading in 'binders,\" [>]–[>]Transcontinental Air\n",
      "Transport, [>]Tri-Continental Allied Corporation, [>]Truman, President Harry S,\n",
      "[>]Trust Company of America, [>]Tucker, Dr. Rufus, [>]Union Industrial Bank of\n",
      "Flint, Michigan, [>]–[>]United Corporation, [>], [>], [>], [>]United Founders\n",
      "Corporation, [>], [>], [>], [>], [>]United States Department of\n",
      "CommerceStatistical Abstract of the United States, [>] (n)., [>] (n).United\n",
      "States Department of Health, Education and Welfare: Mortality Statistics, [>]\n",
      "(n).Vital Statistics: Special Reports, [>]United States Steel Corporation, [>],\n",
      "[>], [>], [>], [>]–[>], [>], [>], [>], [>], [>], [>]–[>], [>]; extra dividend,\n",
      "[>]; Whitney's bid, [>]United States Treasury, [>]; Secretary of the Treasury,\n",
      "[>]–[>]Vanderblue, Homer B., [>] (n)., [>] (n).Vauclain, Samuel, [>]Vest ris,\n",
      "[>]Volume of trade. See New York Stock Exchange, Trading VolumeWages, [>], [>],\n",
      "[>], [>]Walter, Mayor James J., [>], [>], [>]Wall Street, [>], [>], [>], [>],\n",
      "[>]–[>], [>], [>], [>]–[>], [>], [>]–[>], [>], [>], [>], and Babson, [>];\n",
      "bigness of, [>]–[>], [>], [>]–[>], [>]–[>]; criticism of, [>], [>]–[>], [>];\n",
      "importance of, [>]; in another crisis, [>]–[>]; and Lawrence, [>]–[>]; loans at\n",
      "[>] per cent, [>]; margin trading, [>]–[>]; new prestige of, [>], [>]; Number\n",
      "[>], [>]; Raskob plan, [>]; relation to government, [>]–[>]; sponsoring\n",
      "investment trusts, [>]; worst day, [>]–[>]. See also New York Stock Exchange,\n",
      "Stock MarketWall Street and Washington, [>], [>] (n)., [>] (n)., [>] (n).Wall\n",
      "Street Journal, [>], [>], [>] (n)., [>], [>], [>]–[>], [>], [>], [>]Warburg,\n",
      "Paul M., [>]Waynoka, Oklahoma, [>]Wells, Sheldon Sinclair, [>]Western Utility\n",
      "Investors, [>]Westinghouse, [>], [>], [>], [>], [>], [>]Whalen, Grover. [>],\n",
      "[>]Wheat market, \"panic,\" [>]White Sewing Machine Company, [>]Whitney, George,\n",
      "[>]Whitney, Richard, [>], [>] (n)– [>], [>] (n)., [>], [>]; appreciation of by\n",
      "Stock Exchange, [>]; arrest, [>]; as a witness, [>]–[>]; buying to halt panic,\n",
      "[>]–[>] [>], [>]; consequences for Stock Exchange, [>]–[>]; misfortunes of,\n",
      "[>]–[>]Wiggin, Albert H– [>], [>], [>]; speculations, [>]–[>]Williams, Harrison,\n",
      "[>], [>], [>] Wilson, Charles E., [>]Wilson, Senator from Indiana, [>]Wilson,\n",
      "Thomas, [>] (n)., [>] (n)., [>] (n)., [>] (n)– [>] (n)., [>] (n).Women\n",
      "investors, [>]–[>]Wool worth, [>]Wright Aeronautic, [>], [>], [>]Young, Owen D.,\n",
      "[>]Young, Roy A., [>], [>]        Footnotes1 U.S. Department of Commerce, Bureau\n",
      "of the Census, Statistical Abstract of the United States, 1944–15.[back]***2\n",
      "Federal Reserve Bulletin, December 1929.[back]***3 Thomas Wilson, Fluctuations\n",
      "in Income and Employment, 3rd ed. (New York: Pitman, 1948), p. 141.[back]***4\n",
      "These details are principally from two articles on the Florida land boom by\n",
      "Homer B. Vanderblue in The Journal of Land and Public Utility Economics, May and\n",
      "August 1927.[back]***5 Only Yesterday (New York: Harper, 1931), p. 280. Other\n",
      "details of the damage resulting from the hurricane are from this still fresh and\n",
      "lively book.[back]***6 Vanderblue, op. cit., p. 114.[back]***7 Allen, op. cit..\n",
      "p. 282.[back]***8 Throughout this book I have used the New York Times industrial\n",
      "averages as the short-hand designation of the level of security prices. This\n",
      "series is the arithmetical, unweighted average of the prices of twenty-five of\n",
      "what the Times describes as \"good, sound stocks with regular price changes and\n",
      "generally active markets.\" The selection of the Times averages in preference to\n",
      "the Dow-Jones or other averages was largely arbitrary. The Times averages are\n",
      "the ones I have watched over the years; they are somewhat more accessible to the\n",
      "non-professional observer than the Dow-Jones averages. Also, while the latter\n",
      "are much better known, they carry in their wake a certain lore of market theory\n",
      "which is irrelevant for present purposes. The industrial rather than the\n",
      "railroad or combined average is cited because industrial stocks were the major\n",
      "focus of speculation and displayed the widest amplitude of movement. Unless\n",
      "there is indication to the contrary, values given are those at the close of the\n",
      "market for the date indicated.[back]***9 Testimony before Senate Committee,\n",
      "quoted by Lionel Robbins, The Great Depression (New York: Macmillan, 1934), p.\n",
      "53.[back]***10 Ibid., p. 53.[back]***11 Allen, op. cit., p. 297.[back]***12\n",
      "Understanding the New York Stock Exchange, 3rd ed. (New York: Stock Exchange,\n",
      "April 1954), p. 2.[back]***13 New Levels in the Stock Market (New York: McGraw-\n",
      "Hill, 1929), p. 9.[back]***14 Ibid., pp. 6–7.[back]***15 The Memoirs of Herbert\n",
      "Hoover: The Great Depression, 1929–1941 (New York: Macmillan, 1952), p.\n",
      "5.[back]***16 Ibid., p. 14.[back]***17 Dice, op. cit., p. 11.[back]***18 Year\n",
      "Book, 1929–1930 (New York: Stock Exchange).[back]***19 The year-end figure was\n",
      "$5,722,258,724. Figures are from the New York Stock Exchange Year Book,\n",
      "1928–1929, and do not include brokers' time loans.[back]***20 Lombard Street,\n",
      "1922 ed. (London: John Murray, 1922), p. 151.[back]***1 The Memoirs of Herbert\n",
      "Hoover, p. 16.[back]***2 Ibid., p. 11.[back]***3 Ibid., p. 9.[back]***4 Ibid.,\n",
      "p. 9, 10.[back]***5 Or the sale or reduction in inventory of commercial\n",
      "paper.[back]***6 Stock Exchange Practices, Report of the Committee on Banking\n",
      "and Currency pursuant to Senate Resolution 84 (Washington, 1934), p.\n",
      "16.[back]***7 Ibid., pp. 13–14.[back]***8 Thomas Wilson, Fluctuations in Income\n",
      "and Employment, p. 147.[back]***9 Ibid., pp. 147–48.[back]***10 Quoted by\n",
      "Mitchell in Stock Exchange Practices, Hearings, Subcommittee, Senate Committee\n",
      "on Banking and Currency—The Pecora Committee, February-March 1933, Pt. 6, p.\n",
      "1817.[back]***11 Barron's, May 6, 1929.[back]***12 Wall Street and Washington\n",
      "(Princeton: Princeton University Press, 1929), p. 3.[back]***13 Ibid., p.\n",
      "v.[back]***14 The visit is described by Earl Sparling, Mystery Men of Wall\n",
      "Street (New York: Greenberg, 1930), pp. 3–8. The authority is dubious, although\n",
      "the author's facts, as distinct from his interpretation, are frequently\n",
      "accurate.[back]***15 Barron's, June 10, 1929.[back]***16 Seymour E. Harris,\n",
      "Twenty Years of Federal Reserve Policy (Cambridge: Harvard University Press,\n",
      "1933), p. 547. I have made much use of this ultra-conservative but very careful\n",
      "account of Federal Reserve policy.[back]***17 Hoover, Memoirs, p. 17.[back]***18\n",
      "Ibid. Mr. Hoover, who is careless of such details, including his dates,\n",
      "describes Whitney as President of the Exchange, which he only later\n",
      "became.[back]***1 Walter Bagehot, Lombard Street, pp. 130, 131.[back]***2 One\n",
      "estimate puts the number at about forty. Cf. Investment Trusts and Investment\n",
      "Companies, Pt. I, Report of the Securities and Exchange Commission (Washington,\n",
      "1939), p. 36.[back]***3 Ibid., p. 36.[back]***4 And would be more accurately\n",
      "described as an investment company or investment corporation. However, I have\n",
      "kept here to the less precise but more customary usage.[back]***5 The estimates\n",
      "in this paragraph are all from Investment Trusts and Investment Companies, Pt.\n",
      "III, Chap. 1, pp. 3, 4.[back]***6 Ibid., Pt. III, Chap. 2, p. 37 ff.[back]***7\n",
      "Ibid., p. 39.[back]***8 Stock Exchange Practices. Report (Washington, 1934), pp.\n",
      "103–4.[back]***9 The Literary Digest, June 1, 1929.[back]***10 Ibid.[back]***11\n",
      "Wall Street and Washington, p. 163.[back]***12 Bernard J. Reis, False Security\n",
      "(New York: Equinox, 1937). pp. 117 ff. and 296.[back]***13 Investment Trusts and\n",
      "Investment Companies, Pt. I, p. 111.[back]***14 Ibid., Pt. I, pp. 61,\n",
      "62.[back]***15 Ibid., Pt. III, Ch. I, p. 53.[back]***16 Assuming they were\n",
      "reasonably orthodox. Bonds and preferred stock. In these days, were issues with\n",
      "an almost infinite variety of conversion and participation rights.[back]***17\n",
      "Part of it shared with Goldman, Sachs, as will be noted presently. The line\n",
      "between a holding company, which has investment in and control of an operating\n",
      "company (or another holding company), and an investment trust or company, which\n",
      "has investment but which is presumed not to have control, is often a shadowy\n",
      "one. The pyramiding of holding companies and concomitant leverage effects was\n",
      "also a striking feature of the period.[back]***18 Investment Trusts and\n",
      "Investment Companies, Pt. III, Ch. 1, pp. 5, 6[back]***19 Ibid., Pt. I, pp.\n",
      "98–100.[back]***20 Reis, op. cit., p. 124.[back]***21 Stock Exchange Practices,\n",
      "Hearings, April-June 1932, Pt. 2, pp. 566, 567.[back]***22 Details here are from\n",
      "Investment Trusts and Investment Companies, Pt. III, Ch. 1, pp. 6 ff. and 17\n",
      "ff.[back]***23 Details on Shenandoah, Blue Ridge, and the Pacific American\n",
      "merger not from the New York Times of the period are from Investment Trusts md\n",
      "Investment Companies, Pt. III, Ch. 1, pp. 5–7.[back]***24 E. H. H. Simmons, The\n",
      "Principal Causes of the Stock Market Crisis of Nineteen Twenty-Nine (address\n",
      "issued in pamphlet form by the New York Stock Exchange, January 1930), p.\n",
      "16.[back]***25 Stock Exchange Practices, Hearings, April-June 1932, Pt. 2, pp.\n",
      "566–67.[back]***1 Estimates are from Stock Exchange Practices, Report\n",
      "(Washington, 1934), p. 8.[back]***2 The Wall Street Journal, September 19,\n",
      "1929.[back]***3 New Levels in the Stock Market, p. 183.[back]***4 New York\n",
      "Times, August 2, 1929.[back]***5 The American Mazagine, June 1929.[back]***6\n",
      "Wall Street and Washington, p. 179. These passages were later quoted editorially\n",
      "by the New York Times and are reproduced in turn from there.[back]***7 The\n",
      "Commercial and Financial Chronicle, March 9. 1929, p. 1444.[back]***8 Alexander\n",
      "Dana Noyes, The Market Place (Boston: Little, Brown, 1938), p. 324.[back]***9\n",
      "Stock Exchange Practices. Hearings, April-June 1932, Pt. 2, pp. 601\n",
      "ff.[back]***10 Ibid., p. 676 ff.[back]***11 Quoted by Allen, Only Yesterday, p.\n",
      "322.[back]***12 Only Yesterday, p. 315.[back]***13 Stock Exchange Practices,\n",
      "Report, 1934, pp. 9, 10.[back]***14 Noyes, op. cit., p. 328.[back]***15 Stock\n",
      "Exchange Practices, Report, 1934, p. 30 ff.[back]***16 Viscount Erleigh, The\n",
      "South Sea Bubble (New York: Putnam, 1933), p. 11.[back]***17 Noyes, op. cit., p.\n",
      "328.[back]***18 The Literary Digest, August 31, 1929.[back]***19 Quoted in The\n",
      "Literary Digest. August 31, 1929.[back]***20 \"One Day in History,\" Harper's\n",
      "Magazine, November 1937.[back]***21 The Commercial and Financial Chronicle,\n",
      "September 7, 1929.[back]***22 Quoted in The Wall Street Journal, September 6,\n",
      "1929.[back]***23 Edward Angly, Oh, Yeah! (New York: Viking, 1931), p.\n",
      "37.[back]***1 Thomas Wilson, Fluctuations in Income and Employment, p.\n",
      "143.[back]***2 Hatry pleaded guilty and early in 1930 was given a long jail\n",
      "sentence.[back]***3 Edwin Lefèvre, \"The Little Fellow in Wall Street,\" The\n",
      "Saturday Evening Post, January 4, 1930.[back]***4 The amounts to be contributed\n",
      "or otherwise committed were never specified. Frederick Lewis Allen (Only\n",
      "Yesterday, pp. 329–30) says that each of the institutions, along with George F.\n",
      "Baker, Jr., of the First National, who later joined the pool, put up $40\n",
      "million. This total—$240 million—seems much too large to be plausible. The New\n",
      "York Times subsequently suggested (March 9, 1938) that the total was some $20 to\n",
      "$30 millions.[back]***5 Op. cit., p. 330.[back]***6 Quotations have normally\n",
      "been rounded to the nearest whole number in this history. The steel quotation on\n",
      "this day seems to call for an exception.[back]***7 This was stated by Garet\n",
      "Garrett in The Saturday Evening Post (December 28, 1929) and it is generally\n",
      "confirmed by Mr. Hoover in his memoirs. According to Mr. Garrett the banker's\n",
      "consortium asked the President for the statement, which suggests that the\n",
      "reassurance, like the support, was tolerably well organized.[back]***1 Only\n",
      "Yesterday, p. 333.[back]***2 Allen, op. cit., p. 334.[back]***3 The Work of the\n",
      "Stock Exchange in the Panic of 1929, an address by Richard Whitney before the\n",
      "Boston Association of Stock Exchange Finns (Boston: June 10, 1930), pp. 16, 17.\n",
      "Whitney's account, below, of the events of October 29 and thereafter is from the\n",
      "same source.[back]***4 Caught Short! A Saga of Wailing Wall Street (New York:\n",
      "Simon and Schuster, 1929 A.C. [After Crash]), p. 31.[back]***5 Investment News,\n",
      "October 18, 1929, p. 538.[back]**** The Registration Area is the part of the\n",
      "country—most of It—wherein causes of death are duly reported. Data are from\n",
      "Vital Statistics: Special Reports, 1–45, 1935 (Washington: Department of\n",
      "Commerce, Bureau of the Census, 1937).[back]***1 I am grateful to the custodians\n",
      "of vital statistics in the Department of Health, Education and Welfare for\n",
      "tracking, down these figures for me. They are from Mortality Statistics, 1929\n",
      "(Washington: Department of Commerce, Bureau of the Census).[back]***2 Stock\n",
      "Exchange Practices, January 1933, Pt. 4, p. 1214 ff. Such sales were heavy for\n",
      "Friday and Saturday, but in the Exchange records of the time the two days are\n",
      "not segregated. Mr. Donald Durant, the highly uninformed American director of\n",
      "Kreuger and Toll, was in Paris at the time of Kreuger's death and cabled the\n",
      "news to the firm of Lee, Higginson and Company, of which he was a partner. The\n",
      "latter company, which was Kreuger's American investment banker, appears to have\n",
      "refrained scrupulously from acting on the news. Ibid., pp. 1215–16.[back]***3\n",
      "December 7, 1929.[back]***4 Lombard Street, page 150.[back]***5 Magazine of Wall\n",
      "Street, December 14, 1929, p. 264. The reference to Foch and the Marne following\n",
      "it from the same source.[back]***6 Both comments are from The Literary Digest,\n",
      "November 30, 1929.[back]***7 July 9, 1932.[back]***8 Frederick Lewis Allen, Only\n",
      "Yesterday, pp. 340–41.[back]***9 Quoted by Edward Angly, Oh, Yeah!, p. 27, from\n",
      "the New York World October 15, 1930.[back]***1 Quotations are from the Weekly\n",
      "Letters of the date given.[back]***2 New York Herald Tribune, November 3, 1929.\n",
      "Quoted by The Commercial and Financial Chronicle, November 9, 1929.[back]***3\n",
      "New York: Macmillan, 1930. The quotations following are on pages 53 and\n",
      "269.[back]***4 Stock Exchange Practices, Report, 1934, p. 201–2.[back]***5 Stock\n",
      "Exchange Practices, Hearings, October-November 1933, Pt. 6, pp. 2877\n",
      "ff.[back]***6 Stock Exchange Practices, Report, 1934, pp. 192–93.[back]***7\n",
      "Stock Exchange Practices, Report, 1934, pp. 188 ff.[back]***8 Stock Exchange\n",
      "Practices, Hearings, October 1933, Pt. 5, p. 2304.[back]***9 Mr. Aldrich later\n",
      "told a Senate committee (Ibid., p. 4020) that his differences of opinion with\n",
      "friends of Mr. Wiggin, and presumably also with Mr. Wiggin, were a matter of\n",
      "general knowledge.[back]***10 Ibid., p. 2302.[back]***11 Investment News,\n",
      "November 16, 1929, p. 546.[back]***12 Stock Exchange Practices, Report, 1934, p.\n",
      "206.[back]***13 Ibid., p. 322.[back]***14 Ibid., pp. 321, 322.[back]***15 Stock\n",
      "Exchange Practices, Hearings, April 1932, Pt I, p. 1 ff.[back]***16 Stock\n",
      "Exchange Practices, Hearings, February-March 1933, Pt. 6, pp. 2235\n",
      "ff.[back]***17 These details are from Securities and Exchange Commission in the\n",
      "Matter of Richard Whitney, Edwin D. Morgan, Etc., Vol. I, Report on\n",
      "Investigation (Washington, 1938).[back]***18 Securities and Exchange Commission,\n",
      "op. cit., Vol. II. p. 50.[back]***19 The New York Stock Exchange, an address by\n",
      "Richard Whitney before the Industrial Club of St. Louis and the Chamber of\n",
      "Commerce of St. Louis (St. Louis, September 27, 1932).[back]***20 Securities and\n",
      "Exchange Commission, op. cit., Transcript of Hearings, Vol. II, pp. 822,\n",
      "823.[back]***1 Economic Indicators: Historical and Descriptive Supplement, Joint\n",
      "Committee on the Economic Report (Washington, 1953).[back]***2 New Levels in the\n",
      "Stock Market, p. 257.[back]***3 Walter Bagehot, Lombard Street, p. 130. The\n",
      "quotation from Macaulay. above, is cited by Bagehot, p. 128.[back]***4 \"At\n",
      "present it is less likely that the existence of business cycles will be denied\n",
      "than that their regularity will be exaggerated.\" Wesley Clair Mitchell, Business\n",
      "Cycles and Unemployment (New York: McGraw-Hill, 1923), p. 6.[back]***5 Geoffrey\n",
      "H. Moore, Statistical Indications of Cyclical Revivals and Recessions,\n",
      "Occasional Paper 31, National Bureau of Economic Research, Inc. (New York.\n",
      "1950).[back]***6 H. W. Arndt, The Economic Lessons of the Nineteen-Thirties\n",
      "(London: Oxford, 1944), p. 15.[back]***7 E. M. Hugh-Jones and E. A. Radice, An\n",
      "American Experiment (London: Oxford, 1936), 49. Cited by Arndt, op. cit., p.\n",
      "16.[back]***8 This has been widely noted. See Lionel Robbins, The Great\n",
      "Depression, p. 4, and Thomas Wilson, Fluctuations in Income, p. 154 ff., and J.\n",
      "M. Keynes, A Treatise on Money (New York: Harcourt, Brace, 1930), II, 190\n",
      "ff.[back]***9 Perhaps I may be permitted to enlarge on this in slightly more\n",
      "technical terms. The interruption could as well have been caused by an\n",
      "insufficient rate of increase in consumer spending as by a failure in the\n",
      "greater rate of increase of capital goods spending. Under-consumption and under-\n",
      "investment are the same side of the same coin. And some force is added to this\n",
      "explanation by the fact that spending for one important consumers' durable,\n",
      "namely houses, had been declining for several years and suffered a further\n",
      "substantial drop in 1929. However, the investment function we still suppose to\n",
      "be less stable than the consumption function, even though we are less assured of\n",
      "the stability of the latter than we used to be. And in the present case it seems\n",
      "wise to attach causal significance to the part of the spending which had to\n",
      "maintain the largest rate of increase if total spending were to be\n",
      "uninterrupted. The need to maintain a specific rate of increase in investment\n",
      "outlay is insufficiently emphasized by Mr. Thomas Wilson in his book which I\n",
      "have so frequently cited and to which students of the period are\n",
      "indebted.[back]***10 Selma Goldsmith, George Jaszi, Hyman Kaitz, and Maurice\n",
      "Liebenberg, \"Size Distribution of Income since the Mid-Thirties\" The Review of\n",
      "Economics and Statistics, February 1954, pp. 16, 18.[back]***11 Weekly Letter,\n",
      "November 23, 1929.[back]***12 Compiled from Federal Reserve Bulletin, monthly\n",
      "issues, 1929.[back]***13 U.S. Department of Commerce, Bureau of Foreign and\n",
      "Domestic Commerce, Statistical Abstract of the United States, 1942.[back]***14\n",
      "Stock Exchange Practices, Report, 1934, pp. 220–21.[back]***15 Ibid., p.\n",
      "215.[back]***16 Stock Exchange Practices, Hearings, February-March 1933, Pt. 6,\n",
      "p. 2091 ff.[back]***17 Lawrence Sullivan, Prelude to Panic (Washington:\n",
      "Statesman Press, 1936), p. 20.[back]***18 William Starr Myers and Walter H.\n",
      "Newton, The Hoover Administration; A Documented Narrative (New York: Scribners,\n",
      "1936), pp. 339–40.[back]***19 In warning in 1969 of the then current speculation\n",
      "I managed to attract, in a modest way, this epithet.[back]***20 These data are\n",
      "from Goldsmith, et al., \"Size of Distribution of Income,\" pp. 16, 18.[back]***21\n",
      "Quoted by Herbert Hoover, Memoirs, p. 30.[back]\n",
      "Text Statistics:\n",
      "Word Count: 78191\n",
      "Sentence Count: 3147\n",
      "Average Word Length: 4.08\n",
      "Text copied to clipboard.\n"
     ]
    }
   ],
   "source": [
    "import textwrap\n",
    "import pyperclip\n",
    "import pyperclip\n",
    "import ebooklib \n",
    "from ebooklib import epub\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "def extract_text_from_epub(epub_path, start_page=5, end_page=200, width=80):\n",
    "    book = epub.read_epub(epub_path)\n",
    "    text_content = []\n",
    "\n",
    "    # Get all document items\n",
    "    document_items = [item for item in book.get_items() if item.get_type() == ebooklib.ITEM_DOCUMENT]\n",
    "\n",
    "    # Restrict to the specified page range\n",
    "    for i, item in enumerate(document_items[start_page-1:end_page], start=start_page):\n",
    "        soup = BeautifulSoup(item.get_content(), 'html.parser')\n",
    "        text_content.append(soup.get_text())\n",
    "\n",
    "    # Join all text content and wrap it\n",
    "    full_text = '\\n'.join(text_content)\n",
    "    wrapped_text = textwrap.fill(full_text, width=width)\n",
    "\n",
    "    return wrapped_text\n",
    "\n",
    "# Example usage\n",
    "fname = '/home/ys/Documents/The Great Crash 1929 - John Kenneth Galbraith.epub'\n",
    "text = extract_text_from_epub(fname,start_page=5, end_page=100)\n",
    "print(text)\n",
    "\n",
    "# Generate text statistics\n",
    "stats = text_statistics(text)\n",
    "print(\"Text Statistics:\")\n",
    "print(f\"Word Count: {stats['word_count']}\")\n",
    "print(f\"Sentence Count: {stats['sentence_count']}\")\n",
    "print(f\"Average Word Length: {stats['average_word_length']:.2f}\")\n",
    "\n",
    "pyperclip.copy(text)\n",
    "print(\"Text copied to clipboard.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extracting text from pdf files "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/ys/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Miguel Grinberg\n",
      " Flask \n",
      "Web Development\n",
      "DEVELOPING WEB APPLICATIONS WITH PYTHON\n",
      "2nd Edition\n",
      "\n",
      "\n",
      "Miguel Grinberg\n",
      "Flask Web Development\n",
      "Developing Web Applications with Python\n",
      "SECOND EDITION\n",
      "Boston\n",
      "Farnham\n",
      "Sebastopol\n",
      "Tokyo\n",
      "Beijing\n",
      "Boston\n",
      "Farnham\n",
      "Sebastopol\n",
      "Tokyo\n",
      "Beijing\n",
      "\n",
      "978-1-491-99173-2\n",
      "[LSI]\n",
      "Flask Web Development\n",
      "by Miguel Grinberg\n",
      "Copyright © 2018 Miguel Grinberg. All rights reserved.\n",
      "Printed in the United States of America.\n",
      "Published by O’Reilly Media, Inc., 1005 Gravenstein Highway North, Sebastopol, CA 95472.\n",
      "O’Reilly books may be purchased for educational, business, or sales promotional use. Online editions are\n",
      "also available for most titles (http://oreilly.com/safari). For more information, contact our corporate/insti‐\n",
      "tutional sales department: 800-998-9938 or corporate@oreilly.com.\n",
      "Editor: Allyson MacDonald\n",
      "Production Editor: Colleen Cole\n",
      "Copyeditor: Dwight Ramsey\n",
      "Proofreader: Rachel Head\n",
      "Indexer: Ellen Troutman\n",
      "Interior Designer: David Futato\n",
      "Cover Designer: Randy Comer\n",
      "Illustrator: Rebecca Demarest\n",
      "March 2018:\n",
      " Second Edition\n",
      "Revision History for the Second Edition\n",
      "2018-03-02: First Release\n",
      "See http://oreilly.com/catalog/errata.csp?isbn=9781491991732 for release details.\n",
      "The O’Reilly logo is a registered trademark of O’Reilly Media, Inc. Flask Web Development, the cover\n",
      "image, and related trade dress are trademarks of O’Reilly Media, Inc.\n",
      "While the publisher and the author have used good faith efforts to ensure that the information and\n",
      "instructions contained in this work are accurate, the publisher and the author disclaim all responsibility\n",
      "for errors or omissions, including without limitation responsibility for damages resulting from the use of\n",
      "or reliance on this work. Use of the information and instructions contained in this work is at your own\n",
      "risk. If any code samples or other technology this work contains or describes is subject to open source\n",
      "licenses or the intellectual property rights of others, it is your responsibility to ensure that your use\n",
      "thereof complies with such licenses and/or rights.\n",
      "\n",
      "For Alicia.\n",
      "\n",
      "\n",
      "Table of Contents\n",
      "Preface. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .  xi\n",
      "Part I. \n",
      "Introduction to Flask\n",
      "1. Installation. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .  1\n",
      "Creating the Application Directory                                                                               2\n",
      "Virtual Environments                                                                                                      2\n",
      "Creating a Virtual Environment with Python 3                                                          3\n",
      "Creating a Virtual Environment with Python 2                                                          3\n",
      "Working with a Virtual Environment                                                                           4\n",
      "Installing Python Packages with pip                                                                             5\n",
      "2. Basic Application Structure. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .  7\n",
      "Initialization                                                                                                                      7\n",
      "Routes and View Functions                                                                                            8\n",
      "A Complete Application                                                                                                 9\n",
      "Development Web Server                                                                                             10\n",
      "Dynamic Routes                                                                                                             12\n",
      "Debug Mode                                                                                                                   13\n",
      "Command-Line Options                                                                                              15\n",
      "The Request-Response Cycle                                                                                       17\n",
      "Application and Request Contexts                                                                          17\n",
      "Request Dispatching                                                                                                  18\n",
      "The Request Object                                                                                                    19\n",
      "Request Hooks                                                                                                            20\n",
      "Responses                                                                                                                    21\n",
      "Flask Extensions                                                                                                             23\n",
      "v\n",
      "\n",
      "3. Templates. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .  25\n",
      "The Jinja2 Template Engine                                                                                          26\n",
      "Rendering Templates                                                                                                 26\n",
      "Variables                                                                                                                      27\n",
      "Control Structures                                                                                                      28\n",
      "Bootstrap Integration with Flask-Bootstrap                                                              30\n",
      "Custom Error Pages                                                                                                       33\n",
      "Links                                                                                                                                36\n",
      "Static Files                                                                                                                       37\n",
      "Localization of Dates and Times with Flask-Moment                                              38\n",
      "4. Web Forms. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .  43\n",
      "Configuration                                                                                                                 44\n",
      "Form Classes                                                                                                                   44\n",
      "HTML Rendering of Forms                                                                                         47\n",
      "Form Handling in View Functions                                                                             48\n",
      "Redirects and User Sessions                                                                                         51\n",
      "Message Flashing                                                                                                           53\n",
      "5. Databases. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .  57\n",
      "SQL Databases                                                                                                                57\n",
      "NoSQL Databases                                                                                                          58\n",
      "SQL or NoSQL?                                                                                                              59\n",
      "Python Database Frameworks                                                                                     59\n",
      "Database Management with Flask-SQLAlchemy                                                      61\n",
      "Model Definition                                                                                                           62\n",
      "Relationships                                                                                                                  64\n",
      "Database Operations                                                                                                     66\n",
      "Creating the Tables                                                                                                     66\n",
      "Inserting Rows                                                                                                            66\n",
      "Modifying Rows                                                                                                         68\n",
      "Deleting Rows                                                                                                             68\n",
      "Querying Rows                                                                                                           68\n",
      "Database Use in View Functions                                                                                 71\n",
      "Integration with the Python Shell                                                                                72\n",
      "Database Migrations with Flask-Migrate                                                                   73\n",
      "Creating a Migration Repository                                                                             73\n",
      "Creating a Migration Script                                                                                      74\n",
      "Upgrading the Database                                                                                            75\n",
      "Adding More Migrations                                                                                          76\n",
      "vi \n",
      "| \n",
      "Table of Contents\n",
      "\n",
      "6. Email. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .  79\n",
      "Email Support with Flask-Mail                                                                                    79\n",
      "Sending Email from the Python Shell                                                                     81\n",
      "Integrating Emails with the Application                                                                 81\n",
      "Sending Asynchronous Email                                                                                  83\n",
      "7. Large Application Structure. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .  85\n",
      "Project Structure                                                                                                            85\n",
      "Configuration Options                                                                                                  86\n",
      "Application Package                                                                                                      88\n",
      "Using an Application Factory                                                                                   88\n",
      "Implementing Application Functionality in a Blueprint                                      90\n",
      "Application Script                                                                                                          93\n",
      "Requirements File                                                                                                          93\n",
      "Unit Tests                                                                                                                        94\n",
      "Database Setup                                                                                                               96\n",
      "Running the Application                                                                                              97\n",
      "Part II. \n",
      "Example: A Social Blogging Application\n",
      "8. User Authentication. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .  101\n",
      "Authentication Extensions for Flask                                                                         101\n",
      "Password Security                                                                                                        102\n",
      "Hashing Passwords with Werkzeug                                                                       102\n",
      "Creating an Authentication Blueprint                                                                      105\n",
      "User Authentication with Flask-Login                                                                     107\n",
      "Preparing the User Model for Logins                                                                    107\n",
      "Protecting Routes                                                                                                     108\n",
      "Adding a Login Form                                                                                              109\n",
      "Signing Users In                                                                                                       111\n",
      "Signing Users Out                                                                                                    112\n",
      "Understanding How Flask-Login Works                                                              113\n",
      "Testing Logins                                                                                                           114\n",
      "New User Registration                                                                                                115\n",
      "Adding a User Registration Form                                                                          115\n",
      "Registering New Users                                                                                             117\n",
      "Account Confirmation                                                                                                118\n",
      "Generating Confirmation Tokens with itsdangerous                                         118\n",
      "Sending Confirmation Emails                                                                                120\n",
      "Account Management                                                                                                 125\n",
      "Table of Contents \n",
      "| \n",
      "vii\n",
      "\n",
      "9. User Roles. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .  127\n",
      "Database Representation of Roles                                                                             127\n",
      "Role Assignment                                                                                                          131\n",
      "Role Verification                                                                                                          132\n",
      "10. User Profiles. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .  137\n",
      "Profile Information                                                                                                     137\n",
      "User Profile Page                                                                                                          138\n",
      "Profile Editor                                                                                                                141\n",
      "User-Level Profile Editor                                                                                        141\n",
      "Administrator-Level Profile Editor                                                                       143\n",
      "User Avatars                                                                                                                  146\n",
      "11. Blog Posts. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .  151\n",
      "Blog Post Submission and Display                                                                            151\n",
      "Blog Posts on Profile Pages                                                                                         154\n",
      "Paginating Long Blog Post Lists                                                                                155\n",
      "Creating Fake Blog Post Data                                                                                 155\n",
      "Rendering in Pages                                                                                                  157\n",
      "Adding a Pagination Widget                                                                                  158\n",
      "Rich-Text Posts with Markdown and Flask-PageDown                                         161\n",
      "Using Flask-PageDown                                                                                           162\n",
      "Handling Rich Text on the Server                                                                          164\n",
      "Permanent Links to Blog Posts                                                                                  165\n",
      "Blog Post Editor                                                                                                           167\n",
      "12. Followers. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .  171\n",
      "Database Relationships Revisited                                                                              171\n",
      "Many-to-Many Relationships                                                                                 172\n",
      "Self-Referential Relationships                                                                                174\n",
      "Advanced Many-to-Many Relationships                                                              174\n",
      "Followers on the Profile Page                                                                                     178\n",
      "Querying Followed Posts Using a Database Join                                                    181\n",
      "Showing Followed Posts on the Home Page                                                            183\n",
      "13. User Comments. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .  189\n",
      "Database Representation of Comments                                                                   189\n",
      "Comment Submission and Display                                                                           191\n",
      "Comment Moderation                                                                                                193\n",
      "14. Application Programming Interfaces. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .  199\n",
      "Introduction to REST                                                                                                  199\n",
      "viii \n",
      "| \n",
      "Table of Contents\n",
      "\n",
      "Resources Are Everything                                                                                       200\n",
      "Request Methods                                                                                                      201\n",
      "Request and Response Bodies                                                                                201\n",
      "Versioning                                                                                                                 202\n",
      "RESTful Web Services with Flask                                                                              203\n",
      "Creating an API Blueprint                                                                                      203\n",
      "Error Handling                                                                                                         204\n",
      "User Authentication with Flask-HTTPAuth                                                        206\n",
      "Token-Based Authentication                                                                                  208\n",
      "Serializing Resources to and from JSON                                                              210\n",
      "Implementing Resource Endpoints                                                                       213\n",
      "Pagination of Large Resource Collections                                                            216\n",
      "Testing Web Services with HTTPie                                                                       217\n",
      "Part III. \n",
      "The Last Mile\n",
      "15. Testing. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .  221\n",
      "Obtaining Code Coverage Reports                                                                           221\n",
      "The Flask Test Client                                                                                                   224\n",
      "Testing Web Applications                                                                                       225\n",
      "Testing Web Services                                                                                               228\n",
      "End-to-End Testing with Selenium                                                                           230\n",
      "Is It Worth It?                                                                                                                234\n",
      "16. Performance. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .  237\n",
      "Logging Slow Database Performance                                                                       237\n",
      "Source Code Profiling                                                                                                 239\n",
      "17. Deployment. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .  241\n",
      "Deployment Workflow                                                                                               241\n",
      "Logging of Errors During Production                                                                      242\n",
      "Cloud Deployment                                                                                                      243\n",
      "The Heroku Platform                                                                                                  244\n",
      "Preparing the Application                                                                                       244\n",
      "Testing with Heroku Local                                                                                      253\n",
      "Deploying with git push                                                                                          254\n",
      "Deploying an Upgrade                                                                                            255\n",
      "Docker Containers                                                                                                       256\n",
      "Installing Docker                                                                                                      256\n",
      "Building a Container Image                                                                                   257\n",
      "Running a Container                                                                                               261\n",
      "Table of Contents \n",
      "| \n",
      "ix\n",
      "\n",
      "Inspecting a Running Container                                                                            262\n",
      "Pushing Your Container Image to an External Registry                                    263\n",
      "Using an External Database                                                                                    264\n",
      "Container Orchestration with Docker Compose                                                265\n",
      "Cleaning Up Old Containers and Images                                                             269\n",
      "Using Docker in Production                                                                                  270\n",
      "Traditional Deployments                                                                                            270\n",
      "Server Setup                                                                                                              271\n",
      "Importing Environment Variables                                                                         271\n",
      "Setting Up Logging                                                                                                  272\n",
      "18. Additional Resources. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .  275\n",
      "Using an Integrated Development Environment (IDE)                                         275\n",
      "Finding Flask Extensions                                                                                            276\n",
      "Getting Help                                                                                                                 276\n",
      "Getting Involved with Flask                                                                                       277\n",
      "Index. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . .  279\n",
      "x \n",
      "| \n",
      "Table of Contents\n",
      "\n",
      "Preface\n",
      "Flask stands out from other frameworks because it lets developers take the driver’s\n",
      "seat and have full creative control of their applications. Maybe you have heard the\n",
      "phrase “fighting the framework” before. This happens with most frameworks when\n",
      "you decide to solve a problem with a solution that isn’t the official one. It could be\n",
      "that you want to use a different database engine, or maybe a different method of\n",
      "authenticating users. Deviating from the path set by the framework’s developers will\n",
      "give you lots of headaches.\n",
      "Flask is not like that. Do you like relational databases? Great. Flask supports them all.\n",
      "Maybe you prefer a NoSQL database? No problem at all. Flask works with them too.\n",
      "Want to use your own homegrown database engine? Don’t need a database at all? Still\n",
      "fine. With Flask you can choose the components of your application, or even write\n",
      "your own if that’s what you want. No questions asked!\n",
      "The key to this freedom is that Flask was designed from the start to be extended. It\n",
      "comes with a robust core that includes the basic functionality that all web applica‐\n",
      "tions need and expects the rest to be provided by some of the many third-party exten‐\n",
      "sions in the ecosystem—and, of course, by you.\n",
      "In this book I present my workflow for developing web applications with Flask. I\n",
      "don’t claim this to be the only true way to build applications with this framework. You\n",
      "should take my choices as recommendations and not as gospel.\n",
      "Most software development books provide small and focused code examples that\n",
      "demonstrate the different features of the target technology in isolation, leaving the\n",
      "“glue” code that is necessary to transform these different features into a fully working\n",
      "application to be figured out by the reader. I take a completely different approach. All\n",
      "the examples I present are part of a single application that starts out very simple and\n",
      "is expanded in each successive chapter. This application begins life with just a few\n",
      "lines of code and ends as a nicely featured blogging and social networking applica‐\n",
      "tion.\n",
      "xi\n",
      "\n",
      "Who This Book Is For\n",
      "You should have some level of Python coding experience to make the most of this\n",
      "book. Although the book assumes no previous Flask knowledge, Python concepts\n",
      "such as packages, modules, functions, decorators, and object-oriented programming\n",
      "are assumed to be well understood. Some familiarity with exceptions and diagnosing\n",
      "issues from stack traces will be very useful.\n",
      "While working through the examples in this book, you will spend a great deal of time\n",
      "in the command line. You should feel comfortable using the command line of your\n",
      "operating system.\n",
      "Modern web applications cannot avoid the use of HTML, CSS, and JavaScript. The\n",
      "example application that is developed throughout the book obviously makes use of\n",
      "these, but the book itself does not go into a lot of detail regarding these technologies\n",
      "and how they are used. Some degree of familiarity with these languages is recom‐\n",
      "mended if you intend to develop complete applications without the help of a devel‐\n",
      "oper versed in client-side techniques.\n",
      "I released the companion application to this book as open source on GitHub.\n",
      "Although GitHub makes it possible to download applications as regular ZIP or TAR\n",
      "files, I strongly recommend that you install a Git client and familiarize yourself with\n",
      "source code version control (at least with the basic commands to clone and check out\n",
      "the different versions of the application directly from the repository). The short list of\n",
      "commands that you’ll need is shown in “How to Work with the Example Code” on\n",
      "page xiii. You will want to use version control for your own projects as well, so use\n",
      "this book as an excuse to learn Git!\n",
      "Finally, this book is not a complete and exhaustive reference on the Flask framework.\n",
      "Most features are covered, but you should complement this book with the official\n",
      "Flask documentation.\n",
      "How This Book Is Organized\n",
      "This book is divided into three parts.\n",
      "Part I, Introduction to Flask, explores the basics of web application development with\n",
      "the Flask framework and some of its extensions:\n",
      "• Chapter 1 describes the installation and setup of the Flask framework.\n",
      "• Chapter 2 dives straight into Flask with a basic application.\n",
      "• Chapter 3 introduces the use of templates in Flask applications.\n",
      "• Chapter 4 introduces web forms.\n",
      "• Chapter 5 introduces databases.\n",
      "xii \n",
      "| \n",
      "Preface\n",
      "\n",
      "• Chapter 6 introduces email support.\n",
      "• Chapter 7 presents an application structure that is appropriate for medium and\n",
      "large applications.\n",
      "Part II, Example: A Social Blogging Application, builds Flasky, the open source blog‐\n",
      "ging and social networking application that I developed for this book:\n",
      "• Chapter 8 implements a user authentication system.\n",
      "• Chapter 9 implements user roles and permissions.\n",
      "• Chapter 10 implements user profile pages.\n",
      "• Chapter 11 creates the blogging interface.\n",
      "• Chapter 12 implements followers.\n",
      "• Chapter 13 implements user comments for blog posts.\n",
      "• Chapter 14 implements an application programming interface (API).\n",
      "Part III, The Last Mile, describes some important tasks not directly related to applica‐\n",
      "tion coding that need to be considered before publishing an application:\n",
      "• Chapter 15 describes different unit testing strategies in detail.\n",
      "• Chapter 16 gives an overview of performance analysis techniques.\n",
      "• Chapter 17 describes deployment options for Flask applications, including tradi‐\n",
      "tional, cloud-based, and container-based solutions.\n",
      "• Chapter 18 lists additional resources.\n",
      "How to Work with the Example Code\n",
      "The code examples presented in this book are available for download at https://\n",
      "github.com/miguelgrinberg/flasky.\n",
      "The commit history in this repository was carefully created to match the order in\n",
      "which concepts are presented in the book. The recommended way to work with the\n",
      "code is to check out the commits starting from the oldest, then move forward\n",
      "through the commit list as you make progress with the book. As an alternative, Git‐\n",
      "Hub will also let you download each commit as a ZIP or TAR file.\n",
      "If you decide to use Git to work with the source code, then you need to install the Git\n",
      "client, which you can download from http://git-scm.com. The following command\n",
      "downloads the example code using Git:\n",
      "$ git clone https://github.com/miguelgrinberg/flasky.git\n",
      "Preface \n",
      "| \n",
      "xiii\n",
      "\n",
      "The git clone command installs the source code from GitHub into a flasky2 folder\n",
      "that is created in the current directory. This folder does not contain just source code;\n",
      "a copy of the Git repository with the entire history of changes made to the application\n",
      "is also included.\n",
      "In the first chapter you will be asked to check out the initial release of the application,\n",
      "and then, at the proper places, you will be instructed to move forward in the history.\n",
      "The Git command that lets you move through the change history is git checkout.\n",
      "Here is an example:\n",
      "$ git checkout 1a\n",
      "The 1a referenced in the command is a tag: a named point in the commit history of\n",
      "the project. This repository is tagged according to the chapters of the book, so the 1a\n",
      "tag used in the example sets the application files to the initial version used in Chap‐\n",
      "ter 1. Most chapters have more than one tag associated with them, so, for example,\n",
      "tags 5a, 5b, and so on are incremental versions presented in Chapter 5.\n",
      "When you run a git checkout command as just shown, Git will display a warning\n",
      "message that informs you that you are in a “detached HEAD” state. This means that\n",
      "you are not in any specific code branch that can accept new commits, but instead are\n",
      "looking at a particular commit in the middle of the change history of the project.\n",
      "There is no reason to be alarmed by this message, but you should keep in mind that if\n",
      "you make modifications to any files while in this state, issuing another git checkout\n",
      "is going to fail, because Git will not know what to do with the changes you’ve made.\n",
      "So, to be able to continue working with the project you will need to revert the files\n",
      "that you changed back to their original state. The easiest way to do this is with the git\n",
      "reset command:\n",
      "$ git reset --hard\n",
      "This command will destroy any local changes you have made, so you should save\n",
      "anything you don’t want to lose before you use this command.\n",
      "As well as checking out the source files for a version of the application, at certain\n",
      "points you will need to perform additional setup tasks. For example, in some cases\n",
      "you will need to install new Python packages, or apply updates to the database. You\n",
      "will be told when these are necessary.\n",
      "From time to time, you may want to refresh your local repository from the one on\n",
      "GitHub, where bug fixes and improvements may have been applied. The commands\n",
      "that achieve this are:\n",
      "$ git fetch --all\n",
      "$ git fetch --tags\n",
      "$ git reset --hard origin/master\n",
      "xiv \n",
      "| \n",
      "Preface\n",
      "\n",
      "The git fetch commands are used to update the commit history and the tags in\n",
      "your local repository from the remote one on GitHub, but none of this affects the\n",
      "actual source files, which are updated with the git reset command that follows.\n",
      "Once again, be aware that any time git reset is used you will lose any local changes\n",
      "you have made.\n",
      "Another useful operation is to view all the differences between two versions of the\n",
      "application. This can be very useful to understand a change in detail. From the com‐\n",
      "mand line, the git diff command can do this. For example, to see the difference\n",
      "between revisions 2a and 2b, use:\n",
      "$ git diff 2a 2b\n",
      "The differences are shown as a patch, which is not a very intuitive format to review\n",
      "changes if you are not used to working with patch files. You may find that the graphi‐\n",
      "cal comparisons shown by GitHub are much easier to read. For example, the differ‐\n",
      "ences between revisions 2a and 2b can be viewed on GitHub at https://github.com/\n",
      "miguelgrinberg/flasky/compare/2a...2b.\n",
      "Using Code Examples\n",
      "This book is here to help you get your job done. In general, if example code is offered\n",
      "with this book, you may use it in your programs and documentation. You do not\n",
      "need to contact us for permission unless you’re reproducing a significant portion of\n",
      "the code. For example, writing a program that uses several chunks of code from this\n",
      "book does not require permission. Selling or distributing a CD-ROM of examples\n",
      "from O’Reilly books does require permission. Answering a question by citing this\n",
      "book and quoting example code does not require permission. Incorporating a signifi‐\n",
      "cant amount of example code from this book into your product’s documentation does\n",
      "require permission.\n",
      "We appreciate, but do not require, attribution. An attribution usually includes the\n",
      "title, author, publisher, and ISBN. For example: “Flask Web Development, 2nd Edition,\n",
      "by \n",
      "Miguel \n",
      "Grinberg \n",
      "(O’Reilly). \n",
      "Copyright \n",
      "2018 \n",
      "Miguel \n",
      "Grinberg,\n",
      "978-1-491-99173-2.”\n",
      "If you feel your use of code examples falls outside fair use or the permission given\n",
      "above, feel free to contact us at permissions@oreilly.com.\n",
      "Conventions Used in This Book\n",
      "The following typographical conventions are used in this book:\n",
      "Italic\n",
      "Indicates new terms, URLs, email addresses, filenames, and file extensions.\n",
      "Preface \n",
      "| \n",
      "xv\n",
      "\n",
      "Constant width\n",
      "Used for command-line output and program listings, as well as within para‐\n",
      "graphs to refer to commands and to program elements such as variable or func‐\n",
      "tion names, databases, data types, environment variables, statements, and\n",
      "keywords.\n",
      "Constant width bold\n",
      "Shows commands or other text that should be typed literally by the user.\n",
      "Constant width italic or angle brackets (<>)\n",
      "Indicates text that should be replaced with user-supplied values or by values\n",
      "determined by context.\n",
      "This element signifies a tip or suggestion.\n",
      "This element signifies a general note.\n",
      "This element indicates a warning or caution.\n",
      "O’Reilly Safari\n",
      "Safari (formerly Safari Books Online) is a membership-based\n",
      "training and reference platform for enterprise, government,\n",
      "educators, and individuals.\n",
      "Members have access to thousands of books, training videos, Learning Paths, interac‐\n",
      "tive tutorials, and curated playlists from over 250 publishers, including O’Reilly\n",
      "Media, Harvard Business Review, Prentice Hall Professional, Addison-Wesley Profes‐\n",
      "sional, Microsoft Press, Sams, Que, Peachpit Press, Adobe, Focal Press, Cisco Press,\n",
      "John Wiley & Sons, Syngress, Morgan Kaufmann, IBM Redbooks, Packt, Adobe\n",
      "Press, FT Press, Apress, Manning, New Riders, McGraw-Hill, Jones & Bartlett, and\n",
      "Course Technology, among others.\n",
      "xvi \n",
      "| \n",
      "Preface\n",
      "\n",
      "For more information, please visit http://oreilly.com/safari.\n",
      "How to Contact Us\n",
      "Please address comments and questions concerning this book to the publisher:\n",
      "O’Reilly Media, Inc.\n",
      "1005 Gravenstein Highway North\n",
      "Sebastopol, CA 95472\n",
      "800-998-9938 (in the United States or Canada)\n",
      "707-829-0515 (international or local)\n",
      "707-829-0104 (fax)\n",
      "We have a web page for this book, where we list errata, examples, and any additional\n",
      "information. You can access this page at http://bit.ly/flask-web-dev2.\n",
      "To comment or ask technical questions about this book, send email to bookques‐\n",
      "tions@oreilly.com.\n",
      "For more information about our books, courses, conferences, and news, see our web‐\n",
      "site at http://www.oreilly.com.\n",
      "Find us on Facebook: http://facebook.com/oreilly\n",
      "Follow us on Twitter: http://twitter.com/oreillymedia\n",
      "Watch us on YouTube: http://www.youtube.com/oreillymedia\n",
      "Acknowledgments\n",
      "I could not have written this book alone. I have received a lot of help from family, co-\n",
      "workers, old friends, and new friends I’ve made along the way.\n",
      "I’d like to thank Brendan Kohler for his detailed technical review and for his help in\n",
      "giving shape to the chapter on application programming interfaces. I’m also in debt to\n",
      "David Baumgold, Todd Brunhoff, Cecil Rock, and Matthew Hugues, who reviewed\n",
      "the manuscript at different stages of completion and gave me very useful advice\n",
      "regarding what to cover and how to organize the material.\n",
      "Writing the code examples for this book was a considerable effort. I appreciate the\n",
      "help of Daniel Hofmann, who did a thorough code review of the application and\n",
      "pointed out several improvements. I’m also thankful to my teenage son, Dylan Grin‐\n",
      "berg, who suspended his Minecraft addiction for a few weekends and helped me test\n",
      "the code under several platforms.\n",
      "O’Reilly has a wonderful program called Early Release that allows impatient readers\n",
      "to have access to books while they are being written. Some of my Early Release read‐\n",
      "Preface \n",
      "| \n",
      "xvii\n",
      "\n",
      "ers went the extra mile and engaged in useful conversations regarding their experi‐\n",
      "ence working through the book, leading to significant improvements. I’d like to\n",
      "acknowledge Sundeep Gupta, Dan Caron, Brian Wisti, and Cody Scott in particular\n",
      "for the contributions they’ve made to this book.\n",
      "The staff at O’Reilly Media have always been there for me. Above all I’d like to recog‐\n",
      "nize my wonderful editor, Meghan Blanchette, for her support, advice, and assistance\n",
      "from the very first day we met. Meg made the experience of writing my first book a\n",
      "memorable one.\n",
      "To conclude, I would like to give a big thank you to the awesome Flask community.\n",
      "Additional Thanks for the Second Edition\n",
      "I’d like to thank Ally MacDonald, my editor for the second edition of this book, and\n",
      "also Susan Conant, Rachel Roumeliotis, and the whole team at O’Reilly Media for\n",
      "their continued support.\n",
      "The technical reviewers for this edition did a wonderful job pointing out areas to\n",
      "improve and providing me with new perspectives. I’d like to recognize Lorena Mesa,\n",
      "Diane Chen, and Jesse Smith for their great contributions through their feedback and\n",
      "suggestions. I also greatly appreciate the help of my son, Dylan Grinberg, who pains‐\n",
      "takingly tested all the code examples.\n",
      "xviii \n",
      "| \n",
      "Preface\n",
      "\n",
      "PART I\n",
      "Introduction to Flask\n",
      "\n",
      "\n",
      "CHAPTER 1\n",
      "Installation\n",
      "Flask is a small framework by most standards—small enough to be called a “micro-\n",
      "framework,” and small enough that once you become familiar with it, you will likely\n",
      "be able to read and understand all of its source code.\n",
      "But being small does not mean that it does less than other frameworks. Flask was\n",
      "designed as an extensible framework from the ground up; it provides a solid core\n",
      "with the basic services, while extensions provide the rest. Because you can pick and\n",
      "choose the extension packages that you want, you end up with a lean stack that has\n",
      "no bloat and does exactly what you need.\n",
      "Flask has three main dependencies. The routing, debugging, and Web Server Gateway\n",
      "Interface (WSGI) subsystems come from Werkzeug; the template support is provided\n",
      "by Jinja2; and the command-line integration comes from Click. These dependencies\n",
      "are all authored by Armin Ronacher, the author of Flask.\n",
      "Flask has no native support for accessing databases, validating web forms, authenti‐\n",
      "cating users, or other high-level tasks. These and many other key services most web\n",
      "applications need are available through extensions that integrate with the core pack‐\n",
      "ages. As a developer, you have the power to cherry-pick the extensions that work best\n",
      "for your project, or even write your own if you feel inclined to. This is in contrast\n",
      "with a larger framework, where most choices have been made for you and are hard or\n",
      "sometimes impossible to change.\n",
      "In this chapter, you will learn how to install Flask. The only requirement is a com‐\n",
      "puter with Python installed.\n",
      "1\n",
      "\n",
      "The code examples in this book have been verified to work with\n",
      "Python 3.5 and 3.6. Python 2.7 can also be used if desired, but\n",
      "given that this version of Python is not going to be maintained\n",
      "after the year 2020, it is strongly recommended that you use the 3.x\n",
      "versions.\n",
      "If you plan to use a Microsoft Windows computer to work with the\n",
      "example code, you have to decide if you want to use a “native”\n",
      "approach based on Windows tools, or set up your computer in a\n",
      "way that allows you to adopt the more mainstream Unix-based\n",
      "toolset. The code presented in this book is largely compatible with\n",
      "both approaches. In the few cases where the approaches differ, the\n",
      "Unix solution is followed, and alternatives for Windows are noted.\n",
      "If you decide to follow a Unix workflow, you have a few options. If\n",
      "you are using Windows 10, you can enable the Windows Subsys‐\n",
      "tem for Linux (WSL), which is an officially supported feature that\n",
      "creates an Ubuntu Linux installation that runs alongside the native\n",
      "Windows interface, giving you access to a bash shell and the com‐\n",
      "plete set of Unix-based tools. If WSL is not available on your sys‐\n",
      "tem, another good option is Cygwin, an open-source project that\n",
      "emulates the POSIX subsystem used by Unix and provides ports of\n",
      "a large number of Unix tools.\n",
      "Creating the Application Directory\n",
      "To begin, you need to create the directory that will host the example code, which is\n",
      "available in a GitHub repository. As discussed in “How to Work with the Example\n",
      "Code” on page xiii, the most convenient way to do this is by checking out the code\n",
      "directly from GitHub using a Git client. The following commands download the\n",
      "example code from GitHub and initialize the application to version 1a, which is the\n",
      "initial version you will work with:\n",
      "$ git clone https://github.com/miguelgrinberg/flasky.git\n",
      "$ cd flasky\n",
      "$ git checkout 1a\n",
      "If you prefer not to use Git and instead manually type or copy the code, you can sim‐\n",
      "ply create an empty application directory as follows:\n",
      "$ mkdir flasky\n",
      "$ cd flasky\n",
      "Virtual Environments\n",
      "Now that you have the application directory created, it is time to install Flask. The\n",
      "most convenient way to do that is to use a virtual environment. A virtual environment\n",
      "2 \n",
      "| \n",
      "Chapter 1: Installation\n",
      "\n",
      "is a copy of the Python interpreter into which you can install packages privately,\n",
      "without affecting the global Python interpreter installed in your system.\n",
      "Virtual environments are very useful because they prevent package clutter and ver‐\n",
      "sion conflicts in the system’s Python interpreter. Creating a virtual environment for\n",
      "each project ensures that applications have access only to the packages that they use,\n",
      "while the global interpreter remains neat and clean and serves only as a source from\n",
      "which more virtual environments can be created. As an added benefit, unlike the\n",
      "system-wide Python interpreter, virtual environments can be created and managed\n",
      "without administrator rights.\n",
      "Creating a Virtual Environment with Python 3\n",
      "The creation of virtual environments is an area where Python 3 and Python 2 inter‐\n",
      "preters differ. With Python 3, virtual environments are supported natively by the\n",
      "venv package that is part of the Python standard library.\n",
      "If you are using the stock Python 3 interpreter on an Ubuntu Linux\n",
      "system, the standard venv package is not installed by default. To\n",
      "add it to your system, install the python3-venv package as follows:\n",
      "$ sudo apt-get install python3-venv\n",
      "The command that creates a virtual environment has the following structure:\n",
      "$ python3 -m venv virtual-environment-name\n",
      "The -m venv option runs the venv package from the standard library as a standalone\n",
      "script, passing the desired name as an argument.\n",
      "You are now going to create a virtual environment inside the flasky directory. A com‐\n",
      "monly used convention for virtual environments is to call them venv, but you can use\n",
      "a different name if you prefer. Make sure your current directory is set to flasky, and\n",
      "then run this command:\n",
      "$ python3 -m venv venv\n",
      "After the command completes, you will have a subdirectory with the name venv\n",
      "inside flasky, with a brand-new virtual environment that contains a Python inter‐\n",
      "preter for exclusive use by this project.\n",
      "Creating a Virtual Environment with Python 2\n",
      "Python 2 does not have a venv package. In this version of the Python interpreter, vir‐\n",
      "tual environments are created with the third-party utility virtualenv.\n",
      "Creating a Virtual Environment with Python 3 \n",
      "| \n",
      "3\n",
      "\n",
      "Make sure your current directory is set to flasky, and then use one of the following\n",
      "two commands, depending on your operating system. If you are using Linux or\n",
      "macOS, the command is:\n",
      "$ sudo pip install virtualenv\n",
      "If you are using Microsoft Windows, make sure you open a command prompt win‐\n",
      "dow using the “Run as Administrator” option, and then run this command:\n",
      "$ pip install virtualenv\n",
      "The virtualenv command takes the name of the virtual environment as its argu‐\n",
      "ment. Make sure your current directory is set to flasky, and then run the following\n",
      "command to create a virtual environment called venv:\n",
      "$ virtualenv venv\n",
      "New python executable in venv/bin/python2.7\n",
      "Also creating executable in venv/bin/python\n",
      "Installing setuptools, pip, wheel...done.\n",
      "A subdirectory with the venv name will be created in the current directory, and all\n",
      "files associated with the virtual environment will be inside it.\n",
      "Working with a Virtual Environment\n",
      "When you want to start using a virtual environment, you have to “activate” it. If you\n",
      "are using a Linux or macOS computer, you can activate the virtual environment with\n",
      "this command:\n",
      "$ source venv/bin/activate\n",
      "If you are using Microsoft Windows, the activation command is:\n",
      "$ venv\\Scripts\\activate\n",
      "When a virtual environment is activated, the location of its Python interpreter is\n",
      "added to the PATH environment variable in your current command session, which\n",
      "determines where to look for executable files. To remind you that you have activated\n",
      "a virtual environment, the activation command modifies your command prompt to\n",
      "include the name of the environment:\n",
      "(venv) $\n",
      "After a virtual environment is activated, typing python at the command prompt will\n",
      "invoke the interpreter from the virtual environment instead of the system-wide inter‐\n",
      "preter. If you are using more than one command prompt window, you have to acti‐\n",
      "vate the virtual environment in each of them.\n",
      "4 \n",
      "| \n",
      "Chapter 1: Installation\n",
      "\n",
      "While activating a virtual environment is usually the most conve‐\n",
      "nient option, you can also use a virtual environment without acti‐\n",
      "vating it. For example, you can start a Python console for the venv\n",
      "virtual environment by running venv/bin/python on Linux or\n",
      "macOS, or venv\\Scripts\\python on Microsoft Windows.\n",
      "When you are done working with the virtual environment, type deactivate at the\n",
      "command prompt to restore the PATH environment variable for your terminal session\n",
      "and the command prompt to their original states.\n",
      "Installing Python Packages with pip\n",
      "Python packages are installed with the pip package manager, which is included in all\n",
      "virtual environments. Like the python command, typing pip in a command prompt\n",
      "session will invoke the version of this tool that belongs to the activated virtual envi‐\n",
      "ronment.\n",
      "To install Flask into the virtual environment, make sure the venv virtual environment\n",
      "is activated, and then run the following command:\n",
      "(venv) $ pip install flask\n",
      "When you execute this command, pip will not only install Flask, but also all of its\n",
      "dependencies. You can check what packages are installed in the virtual environment\n",
      "at any time using the pip freeze command:\n",
      "(venv) $ pip freeze\n",
      "click==6.7\n",
      "Flask==0.12.2\n",
      "itsdangerous==0.24\n",
      "Jinja2==2.9.6\n",
      "MarkupSafe==1.0\n",
      "Werkzeug==0.12.2\n",
      "The output of pip freeze includes detailed version numbers for each installed pack‐\n",
      "age. The version numbers that you get are likely going to be different from the ones\n",
      "shown here.\n",
      "You can also verify that Flask was correctly installed by starting the Python inter‐\n",
      "preter and trying to import it:\n",
      "(venv) $ python\n",
      ">>> import flask\n",
      ">>>\n",
      "If no errors appear, you can congratulate yourself: you are ready for the next chapter,\n",
      "where you will write your first web application.\n",
      "Installing Python Packages with pip \n",
      "| \n",
      "5\n",
      "\n",
      "\n",
      "CHAPTER 2\n",
      "Basic Application Structure\n",
      "In this chapter, you will learn about the different parts of a Flask application. You will\n",
      "also write and run your first Flask web application.\n",
      "Initialization\n",
      "All Flask applications must create an application instance. The web server passes all\n",
      "requests it receives from clients to this object for handling, using a protocol called\n",
      "Web Server Gateway Interface (WSGI, pronounced “wiz-ghee”). The application\n",
      "instance is an object of class Flask, usually created as follows:\n",
      "from flask import Flask\n",
      "app = Flask(__name__)\n",
      "The only required argument to the Flask class constructor is the name of the main\n",
      "module or package of the application. For most applications, Python’s __name__ vari‐\n",
      "able is the correct value for this argument.\n",
      "The __name__ argument that is passed to the Flask application con‐\n",
      "structor is a source of confusion among new Flask developers.\n",
      "Flask uses this argument to determine the location of the applica‐\n",
      "tion, which in turn allows it to locate other files that are part of the\n",
      "application, such as images and templates.\n",
      "Later you will learn more complex ways to initialize an application, but for simple\n",
      "applications this is all that is needed.\n",
      "7\n",
      "\n",
      "Routes and View Functions\n",
      "Clients such as web browsers send requests to the web server, which in turn sends\n",
      "them to the Flask application instance. The Flask application instance needs to know\n",
      "what code it needs to run for each URL requested, so it keeps a mapping of URLs to\n",
      "Python functions. The association between a URL and the function that handles it is\n",
      "called a route.\n",
      "The most convenient way to define a route in a Flask application is through the\n",
      "app.route decorator exposed by the application instance. The following example\n",
      "shows how a route is declared using this decorator:\n",
      "@app.route('/')\n",
      "def index():\n",
      "    return '<h1>Hello World!</h1>'\n",
      "Decorators are a standard feature of the Python language. A com‐\n",
      "mon use of decorators is to register functions as handler functions\n",
      "to be invoked when certain events occur.\n",
      "The previous example registers function index() as the handler for the application’s\n",
      "root URL. While the app.route decorator is the preferred method to register view\n",
      "functions, Flask also offers a more traditional way to set up the application routes\n",
      "with the app.add_url_rule() method, which in its most basic form takes three argu‐\n",
      "ments: the URL, the endpoint name, and the view function. The following example\n",
      "uses app.add_url_rule() to register an index() function that is equivalent to the\n",
      "one shown previously:\n",
      "def index():\n",
      "    return '<h1>Hello World!</h1>'\n",
      "app.add_url_rule('/', 'index', index)\n",
      "Functions like index() that handle application URLs are called view functions. If the\n",
      "application is deployed on a server associated with the www.example.com domain\n",
      "name, then navigating to http://www.example.com/ in your browser would trigger\n",
      "index() to run on the server. The return value of this view function is the response\n",
      "the client receives. If the client is a web browser, this response is the document that is\n",
      "displayed to the user in the browser window. A response returned by a view function\n",
      "can be a simple string with HTML content, but it can also take more complex forms,\n",
      "as you will see later.\n",
      "8 \n",
      "| \n",
      "Chapter 2: Basic Application Structure\n",
      "\n",
      "Embedding response strings with HTML code in Python source\n",
      "files leads to code that is difficult to maintain. The examples in this\n",
      "chapter do it only to introduce the concept of responses. You will\n",
      "learn a better way to generate HTML responses in Chapter 3.\n",
      "If you pay attention to how some URLs for services that you use every day are\n",
      "formed, you will notice that many have variable sections. For example, the URL for\n",
      "your Facebook profile page has the format https://www.facebook.com/<your-name>,\n",
      "which includes your username, making it different for each user. Flask supports these\n",
      "types of URLs using a special syntax in the app.route decorator. The following exam‐\n",
      "ple defines a route that has a dynamic component:\n",
      "@app.route('/user/<name>')\n",
      "def user(name):\n",
      "    return '<h1>Hello, {}!</h1>'.format(name)\n",
      "The portion of the route URL enclosed in angle brackets is the dynamic part. Any\n",
      "URLs that match the static portions will be mapped to this route, and when the view\n",
      "function is invoked, the dynamic component will be passed as an argument. In the\n",
      "preceding example, the name argument is used to generate a response that includes a\n",
      "personalized greeting.\n",
      "The dynamic components in routes are strings by default but can also be of different\n",
      "types. For example, the route /user/<int:id> would match only URLs that have an\n",
      "integer in the id dynamic segment, such as /user/123. Flask supports the types\n",
      "string, int, float, and path for routes. The path type is a special string type that can\n",
      "include forward slashes, unlike the string type.\n",
      "A Complete Application\n",
      "In the previous sections you learned about the different parts of a Flask web applica‐\n",
      "tion, and now it is time to write your first one. The hello.py application script shown\n",
      "in Example 2-1 defines an application instance and a single route and view function,\n",
      "as described earlier.\n",
      "Example 2-1. hello.py: A complete Flask application\n",
      "from flask import Flask\n",
      "app = Flask(__name__)\n",
      "@app.route('/')\n",
      "def index():\n",
      "    return '<h1>Hello World!</h1>'\n",
      "A Complete Application \n",
      "| \n",
      "9\n",
      "\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can now run git checkout 2a to check out this version of the\n",
      "application.\n",
      "Development Web Server\n",
      "Flask applications include a development web server that can be started with the\n",
      "flask run command. This command looks for the name of the Python script that\n",
      "contains the application instance in the FLASK_APP environment variable.\n",
      "To start the hello.py application from the previous section, first make sure the virtual\n",
      "environment you created earlier is activated and has Flask installed in it. For Linux\n",
      "and macOS users, start the web server as follows:\n",
      "(venv) $ export FLASK_APP=hello.py\n",
      "(venv) $ flask run\n",
      " * Serving Flask app \"hello\"\n",
      " * Running on http://127.0.0.1:5000/ (Press CTRL+C to quit)\n",
      "For Microsoft Windows users, the only difference is in how the FLASK_APP environ‐\n",
      "ment variable is set:\n",
      "(venv) $ set FLASK_APP=hello.py\n",
      "(venv) $ flask run\n",
      " * Serving Flask app \"hello\"\n",
      " * Running on http://127.0.0.1:5000/ (Press CTRL+C to quit)\n",
      "Once the server starts up, it goes into a loop that accepts requests and services them.\n",
      "This loop continues until you stop the application by pressing Ctrl+C.\n",
      "With the server running, open your web browser and type http://localhost:5000/\n",
      "in the address bar. Figure 2-1 shows what you’ll see after connecting to the applica‐\n",
      "tion.\n",
      "10 \n",
      "| \n",
      "Chapter 2: Basic Application Structure\n",
      "\n",
      "Figure 2-1. hello.py Flask application\n",
      "If you type anything else after the base URL, the application will not know how to\n",
      "handle it and will return an error code 404 to the browser—the familiar error that\n",
      "you get when you navigate to a web page that does not exist.\n",
      "The web server provided by Flask is intended to be used only for\n",
      "development and testing. You will learn about production web\n",
      "servers in Chapter 17.\n",
      "The Flask development web server can also be started program‐\n",
      "matically by invoking the app.run() method. Older versions of\n",
      "Flask that did not have the flask command required the server to\n",
      "be started by running the application’s main script, which had to\n",
      "include the following snippet at the end:\n",
      "if __name__ == '__main__':\n",
      "    app.run()\n",
      "While the flask run command makes this practice unnecessary,\n",
      "the app.run() method can still be useful on certain occasions, such\n",
      "as unit testing, as you will learn in Chapter 15.\n",
      "Development Web Server \n",
      "| \n",
      "11\n",
      "\n",
      "Dynamic Routes\n",
      "The second version of the application, shown in Example 2-2, adds a second route\n",
      "that is dynamic. When you visit the dynamic URL in your browser, you are presented\n",
      "with a personalized greeting that includes the name provided in the URL.\n",
      "Example 2-2. hello.py: Flask application with a dynamic route\n",
      "from flask import Flask\n",
      "app = Flask(__name__)\n",
      "@app.route('/')\n",
      "def index():\n",
      "    return '<h1>Hello World!</h1>'\n",
      "@app.route('/user/<name>')\n",
      "def user(name):\n",
      "    return '<h1>Hello, {}!</h1>'.format(name)\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can now run git checkout 2b to check out this version of the\n",
      "application.\n",
      "To test the dynamic route, make sure the server is running and then navigate to\n",
      "http://localhost:5000/user/Dave. The application will respond with the personalized\n",
      "greeting using the name dynamic argument. Try using different names in the URL to\n",
      "see how the view function always generates the response based on the name given.\n",
      "An example is shown in Figure 2-2.\n",
      "12 \n",
      "| \n",
      "Chapter 2: Basic Application Structure\n",
      "\n",
      "Figure 2-2. Dynamic route\n",
      "Debug Mode\n",
      "Flask applications can optionally be executed in debug mode. In this mode, two very\n",
      "convenient modules of the development server called the reloader and the debugger\n",
      "are enabled by default.\n",
      "When the reloader is enabled, Flask watches all the source code files of your project\n",
      "and automatically restarts the server when any of the files are modified. Having a\n",
      "server running with the reloader enabled is extremely useful during development,\n",
      "because every time you modify and save a source file, the server automatically restarts\n",
      "and picks up the change.\n",
      "The debugger is a web-based tool that appears in your browser when your application\n",
      "raises an unhandled exception. The web browser window transforms into an interac‐\n",
      "tive stack trace that allows you to inspect source code and evaluate expressions in any\n",
      "place in the call stack. You can see how the debugger looks in Figure 2-3.\n",
      "Debug Mode \n",
      "| \n",
      "13\n",
      "\n",
      "Figure 2-3. Flask debugger\n",
      "By default, debug mode is disabled. To enable it, set a FLASK_DEBUG=1 environment\n",
      "variable before invoking flask run:\n",
      "(venv) $ export FLASK_APP=hello.py\n",
      "(venv) $ export FLASK_DEBUG=1\n",
      "(venv) $ flask run\n",
      " * Serving Flask app \"hello\"\n",
      " * Forcing debug mode on\n",
      " * Running on http://127.0.0.1:5000/ (Press CTRL+C to quit)\n",
      " * Restarting with stat\n",
      " * Debugger is active!\n",
      " * Debugger PIN: 273-181-528\n",
      "If you are using Microsoft Windows, use set instead of export to set the environ‐\n",
      "ment variables.\n",
      "14 \n",
      "| \n",
      "Chapter 2: Basic Application Structure\n",
      "\n",
      "If you start your server with the app.run() method, the FLASK_APP\n",
      "and FLASK_DEBUG environment variables are not used. To enable\n",
      "debug mode programmatically, use app.run(debug=True).\n",
      "Never enable debug mode on a production server. The debugger in\n",
      "particular allows the client to request remote code execution, so it\n",
      "makes your production server vulnerable to attacks. As a simple\n",
      "protection measure, the debugger needs to be activated with a PIN,\n",
      "printed to the console by the flask run command.\n",
      "Command-Line Options\n",
      "The flask command supports a number of options. To see what’s available, you can\n",
      "run flask --help or just flask without any arguments:\n",
      "(venv) $ flask --help\n",
      "Usage: flask [OPTIONS] COMMAND [ARGS]...\n",
      "  This shell command acts as general utility script for Flask applications.\n",
      "  It loads the application configured (through the FLASK_APP environment\n",
      "  variable) and then provides commands either provided by the application or\n",
      "  Flask itself.\n",
      "  The most useful commands are the \"run\" and \"shell\" command.\n",
      "  Example usage:\n",
      "    $ export FLASK_APP=hello.py\n",
      "    $ export FLASK_DEBUG=1\n",
      "    $ flask run\n",
      "Options:\n",
      "  --version  Show the flask version\n",
      "  --help     Show this message and exit.\n",
      "Commands:\n",
      "  run     Runs a development server.\n",
      "  shell   Runs a shell in the app context.\n",
      "The flask shell command is used to start a Python shell session in the context of\n",
      "the application. You can use this session to run maintenance tasks or tests, or to\n",
      "debug issues. Actual examples where this command is useful will be presented later,\n",
      "in several chapters.\n",
      "Command-Line Options \n",
      "| \n",
      "15\n",
      "\n",
      "You are already familiar with the flask run command, which, as its name implies,\n",
      "runs the application with the development web server. This command has many\n",
      "options:\n",
      "(venv) $ flask run --help\n",
      "Usage: flask run [OPTIONS]\n",
      "  Runs a local development server for the Flask application.\n",
      "  This local server is recommended for development purposes only but it can\n",
      "  also be used for simple intranet deployments.  By default it will not\n",
      "  support any sort of concurrency at all to simplify debugging.  This can be\n",
      "  changed with the --with-threads option which will enable basic\n",
      "  multithreading.\n",
      "  The reloader and debugger are by default enabled if the debug flag of\n",
      "  Flask is enabled and disabled otherwise.\n",
      "Options:\n",
      "  -h, --host TEXT                 The interface to bind to.\n",
      "  -p, --port INTEGER              The port to bind to.\n",
      "  --reload / --no-reload          Enable or disable the reloader.  By default\n",
      "                                  the reloader is active if debug is enabled.\n",
      "  --debugger / --no-debugger      Enable or disable the debugger.  By default\n",
      "                                  the debugger is active if debug is enabled.\n",
      "  --eager-loading / --lazy-loader\n",
      "                                  Enable or disable eager loading.  By default\n",
      "                                  eager loading is enabled if the reloader is\n",
      "                                  disabled.\n",
      "  --with-threads / --without-threads\n",
      "                                  Enable or disable multithreading.\n",
      "  --help                          Show this message and exit.\n",
      "The --host argument is particularly useful because it tells the web server what net‐\n",
      "work interface to listen to for connections from clients. By default, Flask’s develop‐\n",
      "ment web server listens for connections on localhost, so only connections originating\n",
      "from the computer running the server are accepted. The following command makes\n",
      "the web server listen for connections on the public network interface, enabling other\n",
      "computers in the same network to connect as well:\n",
      "(venv) $ flask run --host 0.0.0.0\n",
      " * Serving Flask app \"hello\"\n",
      " * Running on http://0.0.0.0:5000/ (Press CTRL+C to quit)\n",
      "The web server should now be accessible from any computer in the network at http://\n",
      "a.b.c.d:5000, where a.b.c.d is the IP address of the computer running the server in\n",
      "your network.\n",
      "The --reload, --no-reload, --debugger, and --no-debugger options provide a\n",
      "greater degree of control on top of the debug mode setting. For example, if debug\n",
      "16 \n",
      "| \n",
      "Chapter 2: Basic Application Structure\n",
      "\n",
      "mode is enabled, --no-debugger can be used to turn off the debugger, while keeping\n",
      "debug mode and the reloader enabled.\n",
      "The Request-Response Cycle\n",
      "Now that you have played with a basic Flask application, you might want to know\n",
      "more about how Flask works its magic. The following sections describe some of the\n",
      "design aspects of the framework.\n",
      "Application and Request Contexts\n",
      "When Flask receives a request from a client, it needs to make a few objects available\n",
      "to the view function that will handle it. A good example is the request object, which\n",
      "encapsulates the HTTP request sent by the client.\n",
      "The obvious way in which Flask could give a view function access to the request\n",
      "object is by sending it as an argument, but that would require every single view func‐\n",
      "tion in the application to have an extra argument. Things get more complicated if you\n",
      "consider that the request object is not the only object that view functions might need\n",
      "to access to fulfill a request.\n",
      "To avoid cluttering view functions with lots of arguments that may not always be\n",
      "needed, Flask uses contexts to temporarily make certain objects globally accessible.\n",
      "Thanks to contexts, view functions like the following one can be written:\n",
      "from flask import request\n",
      "@app.route('/')\n",
      "def index():\n",
      "    user_agent = request.headers.get('User-Agent')\n",
      "    return '<p>Your browser is {}</p>'.format(user_agent)\n",
      "Note how in this view function, request is used as if it were a global variable. In real‐\n",
      "ity, request cannot be a global variable; in a multithreaded server several threads can\n",
      "be working on different requests from different clients all at the same time, so each\n",
      "thread needs to see a different object in request. Contexts enable Flask to make cer‐\n",
      "tain variables globally accessible to a thread without interfering with the other\n",
      "threads.\n",
      "A thread is the smallest sequence of instructions that can be man‐\n",
      "aged independently. It is common for a process to have multiple\n",
      "active threads, sometimes sharing resources such as memory or file\n",
      "handles. Multithreaded web servers start a pool of threads and\n",
      "select a thread from the pool to handle each incoming request.\n",
      "The Request-Response Cycle \n",
      "| \n",
      "17\n",
      "\n",
      "There are two contexts in Flask: the application context and the request context.\n",
      "Table 2-1 shows the variables exposed by each of these contexts.\n",
      "Table 2-1. Flask context globals\n",
      "Variable name\n",
      "Context\n",
      "Description\n",
      "current_app Application\n",
      "context\n",
      "The application instance for the active application.\n",
      "g\n",
      "Application\n",
      "context\n",
      "An object that the application can use for temporary storage during the handling of a\n",
      "request. This variable is reset with each request.\n",
      "request\n",
      "Request context\n",
      "The request object, which encapsulates the contents of an HTTP request sent by the\n",
      "client.\n",
      "session\n",
      "Request context\n",
      "The user session, a dictionary that the application can use to store values that are\n",
      "“remembered” between requests.\n",
      "Flask activates (or pushes) the application and request contexts before dispatching a\n",
      "request to the application, and removes them after the request is handled. When the\n",
      "application context is pushed, the current_app and g variables become available to\n",
      "the thread. Likewise, when the request context is pushed, request and session\n",
      "become available as well. If any of these variables are accessed without an active appli‐\n",
      "cation or request context, an error is generated. The four context variables will be\n",
      "covered in detail in this and later chapters, so don’t worry if you don’t understand\n",
      "why they are useful yet.\n",
      "The following Python shell session demonstrates how the application context works:\n",
      ">>> from hello import app\n",
      ">>> from flask import current_app\n",
      ">>> current_app.name\n",
      "Traceback (most recent call last):\n",
      "...\n",
      "RuntimeError: working outside of application context\n",
      ">>> app_ctx = app.app_context()\n",
      ">>> app_ctx.push()\n",
      ">>> current_app.name\n",
      "'hello'\n",
      ">>> app_ctx.pop()\n",
      "In this example, current_app.name fails when there is no application context active\n",
      "but becomes valid once an application context for the application is pushed. Note\n",
      "how an application context is obtained by invoking app.app_context() on the appli‐\n",
      "cation instance.\n",
      "Request Dispatching\n",
      "When the application receives a request from a client, it needs to find out what view\n",
      "function to invoke to service it. For this task, Flask looks up the URL given in the\n",
      "18 \n",
      "| \n",
      "Chapter 2: Basic Application Structure\n",
      "\n",
      "request in the application’s URL map, which contains a mapping of URLs to the view\n",
      "functions that handle them. Flask builds this map using the data provided in the\n",
      "app.route decorator, or the equivalent non-decorator version, app.add_url_rule().\n",
      "To see what the URL map in a Flask application looks like, you can inspect the map\n",
      "created for hello.py in the Python shell. Before you try this, make sure that your vir‐\n",
      "tual environment is activated:\n",
      "(venv) $ python\n",
      ">>> from hello import app\n",
      ">>> app.url_map\n",
      "Map([<Rule '/' (HEAD, OPTIONS, GET) -> index>,\n",
      " <Rule '/static/<filename>' (HEAD, OPTIONS, GET) -> static>,\n",
      " <Rule '/user/<name>' (HEAD, OPTIONS, GET) -> user>])\n",
      "The / and /user/<name> routes were defined by the app.route decorators in the\n",
      "application. The /static/<filename> route is a special route added by Flask to give\n",
      "access to static files. You will learn more about static files in Chapter 3.\n",
      "The (HEAD, OPTIONS, GET) elements shown in the URL map are the request methods\n",
      "that are handled by the routes. The HTTP specification defines that all requests are\n",
      "issued with a method, which normally indicates what action the client is asking the\n",
      "server to perform. Flask attaches methods to each route so that different request\n",
      "methods sent to the same URL can be handled by different view functions. The HEAD\n",
      "and OPTIONS methods are managed automatically by Flask, so in practice it can be\n",
      "said that in this application the three routes in the URL map are attached to the GET\n",
      "method, which is used when the client wants to request information such as a web\n",
      "page. You will learn how to create routes for other request methods in Chapter 4.\n",
      "The Request Object\n",
      "You have seen that Flask exposes a request object as a context variable named\n",
      "request. This is an extremely useful object that contains all the information that the\n",
      "client included in the HTTP request. Table 2-2 enumerates the most commonly used\n",
      "attributes and methods of the Flask request object.\n",
      "Table 2-2. Flask request object\n",
      "Attribute or Method\n",
      "Description\n",
      "form\n",
      "A dictionary with all the form fields submitted with the request.\n",
      "args\n",
      "A dictionary with all the arguments passed in the query string of the URL.\n",
      "values\n",
      "A dictionary that combines the values in form and args.\n",
      "cookies\n",
      "A dictionary with all the cookies included in the request.\n",
      "headers\n",
      "A dictionary with all the HTTP headers included in the request.\n",
      "files\n",
      "A dictionary with all the file uploads included with the request.\n",
      "The Request-Response Cycle \n",
      "| \n",
      "19\n",
      "\n",
      "Attribute or Method Description\n",
      "get_data()\n",
      "Returns the buffered data from the request body.\n",
      "get_json()\n",
      "Returns a Python dictionary with the parsed JSON included in the body of the request.\n",
      "blueprint\n",
      "The name of the Flask blueprint that is handling the request. Blueprints are introduced in Chapter 7.\n",
      "endpoint\n",
      "The name of the Flask endpoint that is handling the request. Flask uses the name of the view function\n",
      "as the endpoint name for a route.\n",
      "method\n",
      "The HTTP request method, such as GET or POST.\n",
      "scheme\n",
      "The URL scheme (http or https).\n",
      "is_secure()\n",
      "Returns True if the request came through a secure (HTTPS) connection.\n",
      "host\n",
      "The host defined in the request, including the port number if given by the client.\n",
      "path\n",
      "The path portion of the URL.\n",
      "query_string\n",
      "The query string portion of the URL, as a raw binary value.\n",
      "full_path\n",
      "The path and query string portions of the URL.\n",
      "url\n",
      "The complete URL requested by the client.\n",
      "base_url\n",
      "Same as url, but without the query string component.\n",
      "remote_addr\n",
      "The IP address of the client.\n",
      "environ\n",
      "The raw WSGI environment dictionary for the request.\n",
      "Request Hooks\n",
      "Sometimes it is useful to execute code before or after each request is processed. For\n",
      "example, at the start of each request it may be necessary to create a database connec‐\n",
      "tion or authenticate the user making the request. Instead of duplicating the code that\n",
      "performs these actions in every view function, Flask gives you the option to register\n",
      "common functions to be invoked before or after a request is dispatched.\n",
      "Request hooks are implemented as decorators. These are the four hooks supported by\n",
      "Flask:\n",
      "before_request\n",
      "Registers a function to run before each request.\n",
      "before_first_request\n",
      "Registers a function to run only before the first request is handled. This can be a\n",
      "convenient way to add server initialization tasks.\n",
      "after_request\n",
      "Registers a function to run after each request, but only if no unhandled excep‐\n",
      "tions occurred.\n",
      "teardown_request\n",
      "Registers a function to run after each request, even if unhandled exceptions\n",
      "occurred.\n",
      "20 \n",
      "| \n",
      "Chapter 2: Basic Application Structure\n",
      "\n",
      "A common pattern to share data between request hook functions and view functions\n",
      "is to use the g context global as storage. For example, a before_request handler can\n",
      "load the logged-in user from the database and store it in g.user. Later, when the view\n",
      "function is invoked, it can retrieve the user from there.\n",
      "Examples of request hooks will be shown in future chapters, so don’t worry if the pur‐\n",
      "pose of these hooks does not quite make sense yet.\n",
      "Responses\n",
      "When Flask invokes a view function, it expects its return value to be the response to\n",
      "the request. In most cases the response is a simple string that is sent back to the client\n",
      "as an HTML page.\n",
      "But the HTTP protocol requires more than a string as a response to a request. A very\n",
      "important part of the HTTP response is the status code, which Flask by default sets to\n",
      "200, the code that indicates that the request was carried out successfully.\n",
      "When a view function needs to respond with a different status code, it can add the\n",
      "numeric code as a second return value after the response text. For example, the fol‐\n",
      "lowing view function returns a 400 status code, the code for a bad request error:\n",
      "@app.route('/')\n",
      "def index():\n",
      "    return '<h1>Bad Request</h1>', 400\n",
      "Responses returned by view functions can also take a third argument, a dictionary of\n",
      "headers that are added to the HTTP response. You will see an example of custom\n",
      "response headers in Chapter 14.\n",
      "Instead of returning one, two, or three values as a tuple, Flask view functions have the\n",
      "option of returning a response object. The make_response() function takes one, two,\n",
      "or three arguments, the same values that can be returned from a view function, and\n",
      "returns an equivalent response object. Sometimes it is useful to generate the response\n",
      "object inside the view function, and then use its methods to further configure the\n",
      "response. The following example creates a response object and then sets a cookie in it:\n",
      "from flask import make_response\n",
      "@app.route('/')\n",
      "def index():\n",
      "    response = make_response('<h1>This document carries a cookie!</h1>')\n",
      "    response.set_cookie('answer', '42')\n",
      "    return response\n",
      "Table 2-3 shows the most commonly used attributes and methods available in\n",
      "response objects.\n",
      "The Request-Response Cycle \n",
      "| \n",
      "21\n",
      "\n",
      "Table 2-3. Flask response object\n",
      "Attribute or Method\n",
      "Description\n",
      "status_code\n",
      "The numeric HTTP status code\n",
      "headers\n",
      "A dictionary-like object with all the headers that will be sent with the response\n",
      "set_cookie()\n",
      "Adds a cookie to the response\n",
      "delete_cookie()\n",
      "Removes a cookie\n",
      "content_length\n",
      "The length of the response body\n",
      "content_type\n",
      "The media type of the response body\n",
      "set_data()\n",
      "Sets the response body as a string or bytes value\n",
      "get_data()\n",
      "Gets the response body\n",
      "There is a special type of response called a redirect. This response does not include a\n",
      "page document; it just gives the browser a new URL to navigate to. A very common\n",
      "use of redirects is when working with web forms, as you will learn in Chapter 4.\n",
      "A redirect is typically indicated with a 302 response status code and the URL to go to\n",
      "given in a Location header. A redirect response can be generated manually with a\n",
      "three-value return or with a response object, but given its frequent use, Flask provides\n",
      "a redirect() helper function that creates this type of response:\n",
      "from flask import redirect\n",
      "@app.route('/')\n",
      "def index():\n",
      "    return redirect('http://www.example.com')\n",
      "Another special response is issued with the abort() function, which is used for error\n",
      "handling. The following example returns status code 404 if the id dynamic argument\n",
      "given in the URL does not represent a valid user:\n",
      "from flask import abort\n",
      "@app.route('/user/<id>')\n",
      "def get_user(id):\n",
      "    user = load_user(id)\n",
      "    if not user:\n",
      "        abort(404)\n",
      "    return '<h1>Hello, {}</h1>'.format(user.name)\n",
      "Note that abort() does not return control back to the function because it raises an\n",
      "exception.\n",
      "22 \n",
      "| \n",
      "Chapter 2: Basic Application Structure\n",
      "\n",
      "Flask Extensions\n",
      "Flask is designed to be extended. It intentionally stays out of areas of important func‐\n",
      "tionality such as database and user authentication, giving you the freedom to select\n",
      "the packages that fit your application the best, or to write your own if you so desire.\n",
      "A great variety of Flask extensions for many different purposes have been created by\n",
      "the community, and if that is not enough, any standard Python package or library can\n",
      "be used as well. You will use your first Flask extension in Chapter 3.\n",
      "This chapter introduced the concept of responses to requests, but there is a lot more\n",
      "to say about responses. Flask provides very good support for generating responses\n",
      "using templates, and this is such an important topic that the next chapter is dedicated\n",
      "to it.\n",
      "Flask Extensions \n",
      "| \n",
      "23\n",
      "\n",
      "\n",
      "CHAPTER 3\n",
      "Templates\n",
      "The key to writing applications that are easy to maintain is to write clean and well-\n",
      "structured code. The examples that you have seen so far are too simple to demon‐\n",
      "strate this, but Flask view functions have two completely independent purposes\n",
      "disguised as one, which creates a problem.\n",
      "The obvious task of a view function is to generate a response to a request, as you have\n",
      "seen in the examples shown in Chapter 2. For the simplest requests this is enough,\n",
      "but in many cases a request also triggers a change in the state of the application, and\n",
      "the view function is where this change is generated.\n",
      "For example, consider a user who is registering a new account on a website. The user\n",
      "types an email address and a password in a web form and clicks the Submit button.\n",
      "On the server, a request with the data provided by the user arrives, and Flask dis‐\n",
      "patches it to the view function that handles registration requests. This view function\n",
      "needs to talk to the database to get the new user added, and then generate a response\n",
      "to send back to the browser that includes a success or failure message. These two\n",
      "types of tasks are formally called business logic and presentation logic, respectively.\n",
      "Mixing business and presentation logic leads to code that is hard to understand and\n",
      "maintain. Imagine having to build the HTML code for a large table by concatenating\n",
      "data obtained from the database with the necessary HTML string literals. Moving the\n",
      "presentation logic into templates helps improve the maintainability of the application.\n",
      "A template is a file that contains the text of a response, with placeholder variables for\n",
      "the dynamic parts that will be known only in the context of a request. The process\n",
      "that replaces the variables with actual values and returns a final response string is\n",
      "called rendering. For the task of rendering templates, Flask uses a powerful template\n",
      "engine called Jinja2.\n",
      "25\n",
      "\n",
      "The Jinja2 Template Engine\n",
      "In its simplest form, a Jinja2 template is a file that contains the text of a response.\n",
      "Example 3-1 shows a Jinja2 template that matches the response of the index() view\n",
      "function of Example 2-1.\n",
      "Example 3-1. templates/index.html: Jinja2 template\n",
      "<h1>Hello World!</h1>\n",
      "The response returned by the user() view function of Example 2-2 has a dynamic\n",
      "component, which is represented by a variable. Example 3-2 shows the template that\n",
      "implements this response.\n",
      "Example 3-2. templates/user.html: Jinja2 template\n",
      "<h1>Hello, {{ name }}!</h1>\n",
      "Rendering Templates\n",
      "By default Flask looks for templates in a templates subdirectory located inside the\n",
      "main application directory. For the next version of hello.py, you need to create the\n",
      "templates subdirectory and store the templates defined in the previous examples in it\n",
      "as index.html and user.html, respectively.\n",
      "The view functions in the application need to be modified to render these templates.\n",
      "Example 3-3 shows these changes.\n",
      "Example 3-3. hello.py: rendering a template\n",
      "from flask import Flask, render_template\n",
      "# ...\n",
      "@app.route('/')\n",
      "def index():\n",
      "    return render_template('index.html')\n",
      "@app.route('/user/<name>')\n",
      "def user(name):\n",
      "    return render_template('user.html', name=name)\n",
      "26 \n",
      "| \n",
      "Chapter 3: Templates\n",
      "\n",
      "The function render_template() provided by Flask integrates the Jinja2 template\n",
      "engine with the application. This function takes the filename of the template as its\n",
      "first argument. Any additional arguments are key-value pairs that represent actual\n",
      "values for variables referenced in the template. In this example, the second template is\n",
      "receiving a name variable.\n",
      "Keyword arguments like name=name in the previous example are fairly common, but\n",
      "they may seem confusing and hard to understand if you are not used to them. The\n",
      "“name” on the left side represents the argument name, which is used in the place‐\n",
      "holder written in the template. The “name” on the right side is a variable in the cur‐\n",
      "rent scope that provides the value for the argument of the same name. While this is a\n",
      "common pattern, using the same variable name on both sides is not required.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 3a to check out this version of the applica‐\n",
      "tion.\n",
      "Variables\n",
      "The {{ name }} construct used in the template shown in Example 3-2 references a\n",
      "variable, a special placeholder that tells the template engine that the value that goes in\n",
      "that place should be obtained from data provided at the time the template is ren‐\n",
      "dered.\n",
      "Jinja2 recognizes variables of any type, even complex types such as lists, dictionaries,\n",
      "and objects. The following are some more examples of variables used in templates:\n",
      "<p>A value from a dictionary: {{ mydict['key'] }}.</p>\n",
      "<p>A value from a list: {{ mylist[3] }}.</p>\n",
      "<p>A value from a list, with a variable index: {{ mylist[myintvar] }}.</p>\n",
      "<p>A value from an object's method: {{ myobj.somemethod() }}.</p>\n",
      "Variables can be modified with filters, which are added after the variable name with a\n",
      "pipe character as separator. For example, the following template shows the name vari‐\n",
      "able capitalized:\n",
      "Hello, {{ name|capitalize }}\n",
      "Table 3-1 lists some of the commonly used filters that come with Jinja2.\n",
      "The Jinja2 Template Engine \n",
      "| \n",
      "27\n",
      "\n",
      "Table 3-1. Jinja2 variable filters\n",
      "Filter name\n",
      "Description\n",
      "safe\n",
      "Renders the value without applying escaping\n",
      "capitalize\n",
      "Converts the first character of the value to uppercase and the rest to lowercase\n",
      "lower\n",
      "Converts the value to lowercase characters\n",
      "upper\n",
      "Converts the value to uppercase characters\n",
      "title\n",
      "Capitalizes each word in the value\n",
      "trim\n",
      "Removes leading and trailing whitespace from the value\n",
      "striptags\n",
      "Removes any HTML tags from the value before rendering\n",
      "The safe filter is interesting to highlight. By default Jinja2 escapes all variables for\n",
      "security purposes. For example, if a variable is set to the value '<h1>Hello</h1>',\n",
      "Jinja2 will render the string as '&lt;h1&gt;Hello&lt;/h1&gt;', which will cause the\n",
      "h1 element to be displayed and not interpreted by the browser. Many times it is neces‐\n",
      "sary to display HTML code stored in variables, and for those cases the safe filter is\n",
      "used.\n",
      "Never use the safe filter on values that aren’t trusted, such as text\n",
      "entered by users on web forms.\n",
      "The complete list of filters can be obtained from the official Jinja2 documentation.\n",
      "Control Structures\n",
      "Jinja2 offers several control structures that can be used to alter the flow of the tem‐\n",
      "plate. This section introduces some of the most useful ones with simple examples.\n",
      "The following example shows how conditional statements can be entered in a\n",
      "template:\n",
      "{% if user %}\n",
      "    Hello, {{ user }}!\n",
      "{% else %}\n",
      "    Hello, Stranger!\n",
      "{% endif %}\n",
      "Another common need in templates is to render a list of elements. This example\n",
      "shows how this can be done with a for loop:\n",
      "<ul>\n",
      "    {% for comment in comments %}\n",
      "28 \n",
      "| \n",
      "Chapter 3: Templates\n",
      "\n",
      "        <li>{{ comment }}</li>\n",
      "    {% endfor %}\n",
      "</ul>\n",
      "Jinja2 also supports macros, which are similar to functions in Python code. For\n",
      "example:\n",
      "{% macro render_comment(comment) %}\n",
      "    <li>{{ comment }}</li>\n",
      "{% endmacro %}\n",
      "<ul>\n",
      "    {% for comment in comments %}\n",
      "        {{ render_comment(comment) }}\n",
      "    {% endfor %}\n",
      "</ul>\n",
      "To make macros more reusable, they can be stored in standalone files that are then\n",
      "imported from all the templates that need them:\n",
      "{% import 'macros.html' as macros %}\n",
      "<ul>\n",
      "    {% for comment in comments %}\n",
      "        {{ macros.render_comment(comment) }}\n",
      "    {% endfor %}\n",
      "</ul>\n",
      "Portions of template code that need to be repeated in several places can be stored in a \n",
      "separate file and included from all the templates to avoid repetition:\n",
      "{% include 'common.html' %}\n",
      "Yet another powerful way to reuse is through template inheritance, which is similar to\n",
      "class inheritance in Python code. First, a base template is created with the name\n",
      "base.html:\n",
      "<html>\n",
      "<head>\n",
      "    {% block head %}\n",
      "    <title>{% block title %}{% endblock %} - My Application</title>\n",
      "    {% endblock %}\n",
      "</head>\n",
      "<body>\n",
      "    {% block body %}\n",
      "    {% endblock %}\n",
      "</body>\n",
      "</html>\n",
      "Base templates define blocks that can be overridden by derived templates. The Jinja2\n",
      "block and endblock directives define blocks of content that are added to the base\n",
      "template. In this example, there are blocks called head, title, and body; note that\n",
      "title is contained by head. The following example is a derived template of the base\n",
      "template:\n",
      "The Jinja2 Template Engine \n",
      "| \n",
      "29\n",
      "\n",
      "{% extends \"base.html\" %}\n",
      "{% block title %}Index{% endblock %}\n",
      "{% block head %}\n",
      "    {{ super() }}\n",
      "    <style>\n",
      "    </style>\n",
      "{% endblock %}\n",
      "{% block body %}\n",
      "<h1>Hello, World!</h1>\n",
      "{% endblock %}\n",
      "The extends directive declares that this template derives from base.html. This direc‐\n",
      "tive is followed by new definitions for the three blocks defined in the base template,\n",
      "which are inserted in the proper places. When a block has some content in both the\n",
      "base and derived templates, the content from the derived template is used. Within\n",
      "this block, the derived template can call super() to reference the contents of the\n",
      "block in the base template. In the preceding example, this is done in the head block.\n",
      "Real-world usage of all the control structures presented in this section will be shown\n",
      "later, so you will have the opportunity to see how they work.\n",
      "Bootstrap Integration with Flask-Bootstrap\n",
      "Bootstrap is an open-source web browser framework from Twitter that provides user\n",
      "interface components that help create clean and attractive web pages that are compat‐\n",
      "ible with all modern web browsers used on desktop and mobile platforms.\n",
      "Bootstrap is a client-side framework, so the server is not directly involved with it. All\n",
      "the server needs to do is provide HTML responses that reference Bootstrap’s Cascad‐\n",
      "ing Style Sheets (CSS) and JavaScript files, and instantiate the desired user interface\n",
      "elements through HTML, CSS, and JavaScript code. The ideal place to do all this is in\n",
      "templates.\n",
      "The naive approach to integrating Bootstrap with the application is to make all the\n",
      "necessary changes to the HTML templates, following the recommendations given by\n",
      "the Bootstrap documentation. But this is an area where the use of a Flask extension\n",
      "makes an integration task much simpler, while helping keep these changes nicely\n",
      "organized.\n",
      "The extension is called Flask-Bootstrap, and it can be installed with pip:\n",
      "(venv) $ pip install flask-bootstrap\n",
      "Flask extensions are initialized at the same time the application instance is created.\n",
      "Example 3-4 shows the initialization of Flask-Bootstrap.\n",
      "30 \n",
      "| \n",
      "Chapter 3: Templates\n",
      "\n",
      "Example 3-4. hello.py: Flask-Bootstrap initialization\n",
      "from flask_bootstrap import Bootstrap\n",
      "# ...\n",
      "bootstrap = Bootstrap(app)\n",
      "The extension is usually imported from a flask_<name> package, where <name> is the\n",
      "extension name. Most Flask extensions follow one of two consistent patterns for initi‐\n",
      "alization. In Example 3-4, the extension is initialized by passing the application\n",
      "instance as an argument in the constructor. You will learn about a more advanced\n",
      "method to initialize extensions appropriate for larger applications in Chapter 7.\n",
      "Once Flask-Bootstrap is initialized, a base template that includes all the Bootstrap\n",
      "files and general structure is available to the application. The application then takes\n",
      "advantage of Jinja2’s template inheritance to extend this base template. Example 3-5\n",
      "shows a new version of user.html as a derived template.\n",
      "Example 3-5. templates/user.html: template that uses Flask-Bootstrap\n",
      "{% extends \"bootstrap/base.html\" %}\n",
      "{% block title %}Flasky{% endblock %}\n",
      "{% block navbar %}\n",
      "<div class=\"navbar navbar-inverse\" role=\"navigation\">\n",
      "    <div class=\"container\">\n",
      "        <div class=\"navbar-header\">\n",
      "            <button type=\"button\" class=\"navbar-toggle\"\n",
      "             data-toggle=\"collapse\" data-target=\".navbar-collapse\">\n",
      "                <span class=\"sr-only\">Toggle navigation</span>\n",
      "                <span class=\"icon-bar\"></span>\n",
      "                <span class=\"icon-bar\"></span>\n",
      "                <span class=\"icon-bar\"></span>\n",
      "            </button>\n",
      "            <a class=\"navbar-brand\" href=\"/\">Flasky</a>\n",
      "        </div>\n",
      "        <div class=\"navbar-collapse collapse\">\n",
      "            <ul class=\"nav navbar-nav\">\n",
      "                <li><a href=\"/\">Home</a></li>\n",
      "            </ul>\n",
      "        </div>\n",
      "    </div>\n",
      "</div>\n",
      "{% endblock %}\n",
      "{% block content %}\n",
      "<div class=\"container\">\n",
      "    <div class=\"page-header\">\n",
      "        <h1>Hello, {{ name }}!</h1>\n",
      "    </div>\n",
      "Bootstrap Integration with Flask-Bootstrap \n",
      "| \n",
      "31\n",
      "\n",
      "</div>\n",
      "{% endblock %}\n",
      "The Jinja2 extends directive implements the template inheritance by referencing\n",
      "bootstrap/base.html from Flask-Bootstrap. The base template from Flask-Bootstrap\n",
      "provides a skeleton web page that includes all the Bootstrap CSS and JavaScript files.\n",
      "The user.html template defines three blocks called title, navbar, and content. These\n",
      "are all blocks that the base template exports for derived templates to define. The\n",
      "title block is straightforward; its contents will appear between <title> tags in the\n",
      "header of the rendered HTML document. The navbar and content blocks are\n",
      "reserved for the page navigation bar and main content.\n",
      "In this template, the navbar block defines a simple navigation bar using Bootstrap\n",
      "components. The content block has a container <div> with a page header inside. The\n",
      "greeting line that was in the previous version of the template is now inside the page\n",
      "header. Figure 3-1 shows how the application looks with these changes.\n",
      "Figure 3-1. Bootstrap templates\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 3b to check out this version of the applica‐\n",
      "tion. The Flask-Bootstrap package also needs to be installed in your\n",
      "virtual environment. The Bootstrap official documentation is a\n",
      "great learning resource full of copy/paste-ready examples.\n",
      "Flask-Bootstrap’s base.html template defines several other blocks that can be used in\n",
      "derived templates. Table 3-2 shows the complete list of available blocks.\n",
      "32 \n",
      "| \n",
      "Chapter 3: Templates\n",
      "\n",
      "Table 3-2. Flask-Bootstrap’s base template blocks\n",
      "Block name\n",
      "Description\n",
      "doc\n",
      "The entire HTML document\n",
      "html_attribs\n",
      "Attributes inside the <html> tag\n",
      "html\n",
      "The contents of the <html> tag\n",
      "head\n",
      "The contents of the <head> tag\n",
      "title\n",
      "The contents of the <title> tag\n",
      "metas\n",
      "The list of <meta> tags\n",
      "styles\n",
      "CSS definitions\n",
      "body_attribs Attributes inside the <body> tag\n",
      "body\n",
      "The contents of the <body> tag\n",
      "navbar\n",
      "User-defined navigation bar\n",
      "content\n",
      "User-defined page content\n",
      "scripts\n",
      "JavaScript declarations at the bottom of the document\n",
      "Many of the blocks in Table 3-2 are used by Flask-Bootstrap itself, so overriding them\n",
      "directly would cause problems. For example, the styles and scripts blocks are\n",
      "where the Bootstrap CSS and JavaScript files are declared. If the application needs to\n",
      "add its own content to a block that already has some content, then Jinja2’s super()\n",
      "function must be used. For example, this is how the scripts block would need to be\n",
      "written in the derived template to add a new JavaScript file to the document:\n",
      "{% block scripts %}\n",
      "{{ super() }}\n",
      "<script type=\"text/javascript\" src=\"my-script.js\"></script>\n",
      "{% endblock %}\n",
      "Custom Error Pages\n",
      "When you enter an invalid route in your browser’s address bar, you get a code 404\n",
      "error page. Compared to the Bootstrap-powered pages, the default error page is now\n",
      "too plain and unattractive, and it has no consistency with the actual pages generated\n",
      "by the application.\n",
      "Flask allows an application to define custom error pages that can be based on tem‐\n",
      "plates, like regular routes. The two most common error codes are 404, triggered when\n",
      "the client requests a page or route that is not known, and 500, triggered when there is\n",
      "an unhandled exception in the application. Example 3-6 shows how to provide cus‐\n",
      "tom handlers for these two errors using the app.errorhandler decorator.\n",
      "Custom Error Pages \n",
      "| \n",
      "33\n",
      "\n",
      "Example 3-6. hello.py: custom error pages\n",
      "@app.errorhandler(404)\n",
      "def page_not_found(e):\n",
      "    return render_template('404.html'), 404\n",
      "@app.errorhandler(500)\n",
      "def internal_server_error(e):\n",
      "    return render_template('500.html'), 500\n",
      "Error handlers return a response, like view functions, but they also need to return the\n",
      "numeric status code that corresponds to the error, which Flask conveniently accepts\n",
      "as a second return value.\n",
      "The templates referenced in the error handlers need to be written. These templates\n",
      "should follow the same layout as the regular pages, so in this case they will have a\n",
      "navigation bar and a page header that shows the error message.\n",
      "The straightforward way to write these templates is to copy templates/user.html to\n",
      "templates/404.html and templates/500.html and then change the page header elements\n",
      "in these two new files to the appropriate error messages, but this will generate a lot of\n",
      "duplication.\n",
      "Jinja2’s template inheritance can help with this. In the same way Flask-Bootstrap pro‐\n",
      "vides a base template with the basic layout of the page, the application can define its\n",
      "own base template with a uniform page layout that includes the navigation bar and\n",
      "leaves the page content to be defined in derived templates. Example 3-7 shows tem‐\n",
      "plates/base.html, a new template that inherits from bootstrap/base.html and defines\n",
      "the navigation bar but is itself a second-level base template to other templates such as\n",
      "templates/user.html, templates/404.html, and templates/500.html.\n",
      "Example 3-7. templates/base.html: base application template with navigation bar\n",
      "{% extends \"bootstrap/base.html\" %}\n",
      "{% block title %}Flasky{% endblock %}\n",
      "{% block navbar %}\n",
      "<div class=\"navbar navbar-inverse\" role=\"navigation\">\n",
      "    <div class=\"container\">\n",
      "        <div class=\"navbar-header\">\n",
      "            <button type=\"button\" class=\"navbar-toggle\"\n",
      "             data-toggle=\"collapse\" data-target=\".navbar-collapse\">\n",
      "                <span class=\"sr-only\">Toggle navigation</span>\n",
      "                <span class=\"icon-bar\"></span>\n",
      "                <span class=\"icon-bar\"></span>\n",
      "                <span class=\"icon-bar\"></span>\n",
      "            </button>\n",
      "            <a class=\"navbar-brand\" href=\"/\">Flasky</a>\n",
      "34 \n",
      "| \n",
      "Chapter 3: Templates\n",
      "\n",
      "        </div>\n",
      "        <div class=\"navbar-collapse collapse\">\n",
      "            <ul class=\"nav navbar-nav\">\n",
      "                <li><a href=\"/\">Home</a></li>\n",
      "            </ul>\n",
      "        </div>\n",
      "    </div>\n",
      "</div>\n",
      "{% endblock %}\n",
      "{% block content %}\n",
      "<div class=\"container\">\n",
      "    {% block page_content %}{% endblock %}\n",
      "</div>\n",
      "{% endblock %}\n",
      "The content block of this template is just a container <div> element that wraps a new\n",
      "empty block called page_content, which derived templates can define.\n",
      "The templates of the application will now inherit from this template instead of\n",
      "directly from Flask-Bootstrap. Example 3-8 shows how simple it is to construct a cus‐\n",
      "tom code 404 error page that inherits from templates/base.html. The page for the 500\n",
      "error is similar, and you can find it in the GitHub repository for the application.\n",
      "Example 3-8. templates/404.html: custom code 404 error page using template\n",
      "inheritance\n",
      "{% extends \"base.html\" %}\n",
      "{% block title %}Flasky - Page Not Found{% endblock %}\n",
      "{% block page_content %}\n",
      "<div class=\"page-header\">\n",
      "    <h1>Not Found</h1>\n",
      "</div>\n",
      "{% endblock %}\n",
      "Figure 3-2 shows how the error page looks in the browser.\n",
      "Custom Error Pages \n",
      "| \n",
      "35\n",
      "\n",
      "Figure 3-2. Custom code 404 error page\n",
      "The templates/user.html template can now be simplified by making it inherit from the\n",
      "base template, as shown in Example 3-9.\n",
      "Example 3-9. templates/user.html: simplified page template using template inheritance\n",
      "{% extends \"base.html\" %}\n",
      "{% block title %}Flasky{% endblock %}\n",
      "{% block page_content %}\n",
      "<div class=\"page-header\">\n",
      "    <h1>Hello, {{ name }}!</h1>\n",
      "</div>\n",
      "{% endblock %}\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 3c to check out this version of the applica‐\n",
      "tion.\n",
      "Links\n",
      "Any application that has more than one route will invariably need to include links\n",
      "that connect the different pages, such as in a navigation bar.\n",
      "Writing the URLs as links directly in the template is trivial for simple routes, but for\n",
      "dynamic routes with variable portions it can get more complicated to build the URLs\n",
      "36 \n",
      "| \n",
      "Chapter 3: Templates\n",
      "\n",
      "right in the template. Also, URLs written explicitly create an unwanted dependency\n",
      "on the routes defined in the code. If the routes are reorganized, links in templates\n",
      "may break.\n",
      "To avoid these problems, Flask provides the url_for() helper function, which gener‐\n",
      "ates URLs from the information stored in the application’s URL map.\n",
      "In its simplest usage, this function takes the view function name (or endpoint name\n",
      "for routes defined with app.add_url_route()) as its single argument and returns its\n",
      "URL. For example, in the current version of hello.py the call url_for('index')\n",
      "would return /, the root URL of the application. Calling url_for('index',\n",
      "_external=True) would instead return an absolute URL, which in this example is\n",
      "http://localhost:5000/.\n",
      "Relative URLs are sufficient when generating links that connect the\n",
      "different routes of the application. Absolute URLs are necessary\n",
      "only for links that will be used outside of the web browser, such as\n",
      "when sending links by email.\n",
      "Dynamic URLs can be generated with url_for() by passing the dynamic parts as\n",
      "keyword \n",
      "arguments. \n",
      "For \n",
      "example, \n",
      "url_for('user', \n",
      "name='john',\n",
      "_external=True) would return http://localhost:5000/user/john.\n",
      "Keyword arguments sent to url_for() are not limited to arguments used by dynamic\n",
      "routes. The function will add any arguments that are not dynamic to the query string.\n",
      "For example, url_for('user', name='john', page=2, version=1) would\n",
      "return /user/john?page=2&version=1.\n",
      "Static Files\n",
      "Web applications are not made of Python code and templates alone. Most applica‐\n",
      "tions also use static files such as images, JavaScript source files, and CSS files that are\n",
      "all referenced from the HTML code in templates.\n",
      "You may recall that when the hello.py application’s URL map was inspected in Chap‐\n",
      "ter 2, a static entry appeared in it. Flask automatically supports static files by adding\n",
      "a special route to the application defined as /static/<filename>. For example, a call\n",
      "to url_for('static', filename='css/styles.css', _external=True) would\n",
      "return http://localhost:5000/static/css/styles.css.\n",
      "In its default configuration, Flask looks for static files in a subdirectory called static\n",
      "located in the application’s root folder. Files can be organized in subdirectories inside\n",
      "this folder if desired. When the server receives a URL that maps to a static route, it\n",
      "Static Files \n",
      "| \n",
      "37\n",
      "\n",
      "generates a response that includes the contents of the corresponding file in the file\n",
      "system.\n",
      "Example 3-10 shows how the application can include a favicon.ico icon in the base\n",
      "template for browsers to show in the address bar.\n",
      "Example 3-10. templates/base.html: favicon definition\n",
      "{% block head %}\n",
      "{{ super() }}\n",
      "<link rel=\"shortcut icon\" href=\"{{ url_for('static', filename='favicon.ico') }}\"\n",
      "    type=\"image/x-icon\">\n",
      "<link rel=\"icon\" href=\"{{ url_for('static', filename='favicon.ico') }}\"\n",
      "    type=\"image/x-icon\">\n",
      "{% endblock %}\n",
      "The icon declaration is inserted at the end of the head block. Note how super() is\n",
      "used to preserve the original contents of the block defined in the base templates.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 3d to check out this version of the applica‐\n",
      "tion.\n",
      "Localization of Dates and Times with Flask-Moment\n",
      "Handling of dates and times in a web application is not a trivial problem when users\n",
      "work in different parts of the world.\n",
      "The server needs uniform time units that are independent of the location of each\n",
      "user, so typically Coordinated Universal Time (UTC) is used. For users, however, see‐\n",
      "ing times expressed in UTC can be confusing, as users always expect to see dates and\n",
      "times presented in their local time and formatted according to the customs of their\n",
      "region.\n",
      "An elegant solution that allows the server to work exclusively in UTC is to send these\n",
      "time units to the web browser, where they are converted to local time and rendered\n",
      "using JavaScript. Web browsers can do a much better job at this task because they\n",
      "have access to time zone and locale settings on the user’s computer.\n",
      "There is an excellent open source library written in JavaScript that renders dates and\n",
      "times in the browser called Moment.js. Flask-Moment is an extension for Flask appli‐\n",
      "cations that makes the integration of Moment.js into Jinja2 templates very easy. Flask-\n",
      "Moment is installed with pip:\n",
      "(venv) $ pip install flask-moment\n",
      "38 \n",
      "| \n",
      "Chapter 3: Templates\n",
      "\n",
      "The extension is initialized in a similar way to Flask-Bootstrap. The required code is\n",
      "shown in Example 3-11.\n",
      "Example 3-11. hello.py: initializing Flask-Moment\n",
      "from flask_moment import Moment\n",
      "moment = Moment(app)\n",
      "Flask-Moment depends on jQuery.js in addition to Moment.js. These two libraries\n",
      "need to be included somewhere in the HTML document—either directly, in which\n",
      "case you can choose what versions to use, or through the helper functions provided\n",
      "by the extension, which reference tested versions of these libraries from a content\n",
      "delivery network (CDN). Because Bootstrap already includes jQuery.js, only\n",
      "Moment.js needs to be added in this case. Example 3-12 shows how this library is\n",
      "loaded in the scripts block of the template, while also preserving the original con‐\n",
      "tents of the block provided by the base template. Note that since this is a predefined\n",
      "block in the Flask-Bootstrap base template, the location in templates/base.html where\n",
      "this block is inserted does not matter.\n",
      "Example 3-12. templates/base.html: importing the Moment.js library\n",
      "{% block scripts %}\n",
      "{{ super() }}\n",
      "{{ moment.include_moment() }}\n",
      "{% endblock %}\n",
      "To work with timestamps, Flask-Moment makes a moment object available to tem‐\n",
      "plates. Example 3-13 demonstrates passing a variable called current_time to the tem‐\n",
      "plate for rendering.\n",
      "Example 3-13. hello.py: adding a datetime variable\n",
      "from datetime import datetime\n",
      "@app.route('/')\n",
      "def index():\n",
      "    return render_template('index.html',\n",
      "                           current_time=datetime.utcnow())\n",
      "Example 3-14 shows how this current_time template variable is rendered.\n",
      "Example 3-14. templates/index.html: timestamp rendering with Flask-Moment\n",
      "<p>The local date and time is {{ moment(current_time).format('LLL') }}.</p>\n",
      "<p>That was {{ moment(current_time).fromNow(refresh=True) }}</p>\n",
      "Localization of Dates and Times with Flask-Moment \n",
      "| \n",
      "39\n",
      "\n",
      "The format('LLL') function renders the date and time according to the time zone\n",
      "and locale settings in the client computer. The argument determines the rendering\n",
      "style, from 'L' to 'LLLL' for four different levels of verbosity. The format() function\n",
      "can also accept a long list of custom format specifiers.\n",
      "The fromNow() render style shown in the second line renders a relative timestamp\n",
      "and automatically refreshes it as time passes. Initially this timestamp will be shown as\n",
      "“a few seconds ago,” but the refresh=True option will keep it updated as time passes,\n",
      "so if you leave the page open for a few minutes you will see the text changing to “a\n",
      "minute ago,” then “2 minutes ago,” and so on.\n",
      "Figure 3-3 shows how the http://localhost:5000/ route looks after the two timestamps\n",
      "are added to the index.html template.\n",
      "Figure 3-3. Page with two Flask-Moment timestamps\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 3e to check out this version of the applica‐\n",
      "tion.\n",
      "Flask-Moment implements the format(), fromNow(), fromTime(), calendar(),\n",
      "valueOf(), and unix() methods from Moment.js. Consult the Moment.js documen‐\n",
      "tation to learn about all the formatting options offered by this library.\n",
      "40 \n",
      "| \n",
      "Chapter 3: Templates\n",
      "\n",
      "Flask-Moment assumes that timestamps handled by the server-side\n",
      "application are “naive” datetime objects expressed in UTC. See the\n",
      "documentation for the datetime package in the standard library\n",
      "for information on naive and aware date and time objects.\n",
      "The timestamps rendered by Flask-Moment can be localized to many languages. A\n",
      "language can be selected in the template by passing the two-letter language code to\n",
      "function locale(), right after the Moment.js library is included. For example, here is\n",
      "how to configure Moment.js to use Spanish:\n",
      "{% block scripts %}\n",
      "{{ super() }}\n",
      "{{ moment.include_moment() }}\n",
      "{{ moment.locale('es') }}\n",
      "{% endblock %}\n",
      "With all the techniques discussed in this chapter, you should be able to build modern\n",
      "and user-friendly web pages for your application. The next chapter touches on an\n",
      "aspect of templates not yet discussed: how to interact with the user through web\n",
      "forms.\n",
      "Localization of Dates and Times with Flask-Moment \n",
      "| \n",
      "41\n",
      "\n",
      "\n",
      "CHAPTER 4\n",
      "Web Forms\n",
      "The templates that you worked with in Chapter 3 are unidirectional, in the sense that\n",
      "they allow information to flow from the server to the user. For most applications,\n",
      "however, there is also a need to have information that flows in the other direction,\n",
      "with the user providing data that the server accepts and processes.\n",
      "With HTML, it is possible to create web forms, in which users can enter information.\n",
      "The form data is then submitted by the web browser to the server, typically in the\n",
      "form of a POST request. The Flask request object, introduced in Chapter 2, exposes all\n",
      "the information sent by the client in a request and, in particular for POST requests\n",
      "containing form data, provides access to the user information through request.form.\n",
      "Although the support provided in Flask’s request object is sufficient for the handling\n",
      "of web forms, there are a number of tasks that can become tedious and repetitive.\n",
      "Two good examples are the generation of HTML code for the forms and the valida‐\n",
      "tion of the submitted form data.\n",
      "The Flask-WTF extension makes working with web forms a much more pleasant\n",
      "experience. This extension is a Flask integration wrapper around the framework-\n",
      "agnostic WTForms package.\n",
      "Flask-WTF and its dependencies can be installed with pip:\n",
      "(venv) $ pip install flask-wtf\n",
      "43\n",
      "\n",
      "Configuration\n",
      "Unlike most other extensions, Flask-WTF does not need to be initialized at the appli‐\n",
      "cation level, but it expects the application to have a secret key configured. A secret key\n",
      "is a string with any random and unique content that is used as an encryption or sign‐\n",
      "ing key to improve the security of the application in several ways. Flask uses this key\n",
      "to protect the contents of the user session against tampering. You should pick a dif‐\n",
      "ferent secret key in each application that you build and make sure that this string is\n",
      "not known by anyone. Example 4-1 shows how to configure a secret key in a Flask\n",
      "application.\n",
      "Example 4-1. hello.py: Flask-WTF configuration\n",
      "app = Flask(__name__)\n",
      "app.config['SECRET_KEY'] = 'hard to guess string'\n",
      "The app.config dictionary is a general-purpose place to store configuration variables\n",
      "used by Flask, extensions, or the application itself. Configuration values can be added\n",
      "to the app.config object using standard dictionary syntax. The configuration object\n",
      "also has methods to import configuration values from files or the environment. A\n",
      "more practical way to manage configuration values for a larger application will be\n",
      "discussed in Chapter 7.\n",
      "Flask-WTF requires a secret key to be configured in the application because this key\n",
      "is part of the mechanism the extension uses to protect all forms against cross-site\n",
      "request forgery (CSRF) attacks. A CSRF attack occurs when a malicious website sends\n",
      "requests to the application server on which the user is currently logged in. Flask-WTF\n",
      "generates security tokens for all forms and stores them in the user session, which is\n",
      "protected with a cryptographic signature generated from the secret key.\n",
      "For added security, the secret key should be stored in an environ‐\n",
      "ment variable instead of being embedded in the source code. This\n",
      "technique is described in Chapter 7.\n",
      "Form Classes\n",
      "When using Flask-WTF, each web form is represented in the server by a class that\n",
      "inherits from the class FlaskForm. The class defines the list of fields in the form, each\n",
      "represented by an object. Each field object can have one or more validators attached.\n",
      "A validator is a function that checks whether the data submitted by the user is valid.\n",
      "44 \n",
      "| \n",
      "Chapter 4: Web Forms\n",
      "\n",
      "Example 4-2 shows a simple web form that has a text field and a submit button.\n",
      "Example 4-2. hello.py: form class definition\n",
      "from flask_wtf import FlaskForm\n",
      "from wtforms import StringField, SubmitField\n",
      "from wtforms.validators import DataRequired\n",
      "class NameForm(FlaskForm):\n",
      "    name = StringField('What is your name?', validators=[DataRequired()])\n",
      "    submit = SubmitField('Submit')\n",
      "The fields in the form are defined as class variables, and each class variable is assigned\n",
      "an object associated with the field type. In this example, the NameForm form has a text\n",
      "field called name and a submit button called submit. The StringField class repre‐\n",
      "sents an HTML <input> element with a type=\"text\" attribute. The SubmitField\n",
      "class represents an HTML <input> element with a type=\"submit\" attribute. The first\n",
      "argument to the field constructors is the label that will be used when rendering the\n",
      "form to HTML.\n",
      "The optional validators argument included in the StringField constructor defines\n",
      "a list of checkers that will be applied to the data submitted by the user before it is\n",
      "accepted. The DataRequired() validator ensures that the field is not submitted\n",
      "empty.\n",
      "The FlaskForm base class is defined by the Flask-WTF extension,\n",
      "so it is imported from flask_wtf. The fields and validators, how‐\n",
      "ever, are imported directly from the WTForms package.\n",
      "The list of standard HTML fields supported by WTForms is shown in Table 4-1.\n",
      "Table 4-1. WTForms standard HTML fields\n",
      "Field type\n",
      "Description\n",
      "BooleanField\n",
      "Checkbox with True and False values\n",
      "DateField\n",
      "Text field that accepts a datetime.date value in a given format\n",
      "DateTimeField\n",
      "Text field that accepts a datetime.datetime value in a given format\n",
      "DecimalField\n",
      "Text field that accepts a decimal.Decimal value\n",
      "FileField\n",
      "File upload field\n",
      "HiddenField\n",
      "Hidden text field\n",
      "MultipleFileField\n",
      "Multiple file upload field\n",
      "FieldList\n",
      "List of fields of a given type\n",
      "Form Classes \n",
      "| \n",
      "45\n",
      "\n",
      "Field type\n",
      "Description\n",
      "FloatField\n",
      "Text field that accepts a floating-point value\n",
      "FormField\n",
      "Form embedded as a field in a container form\n",
      "IntegerField\n",
      "Text field that accepts an integer value\n",
      "PasswordField\n",
      "Password text field\n",
      "RadioField\n",
      "List of radio buttons\n",
      "SelectField\n",
      "Drop-down list of choices\n",
      "SelectMultipleField\n",
      "Drop-down list of choices with multiple selection\n",
      "SubmitField\n",
      "Form submission button\n",
      "StringField\n",
      "Text field\n",
      "TextAreaField\n",
      "Multiple-line text field\n",
      "The list of WTForms built-in validators is shown in Table 4-2.\n",
      "Table 4-2. WTForms validators\n",
      "Validator\n",
      "Description\n",
      "DataRequired\n",
      "Validates that the field contains data after type conversion\n",
      "Email\n",
      "Validates an email address\n",
      "EqualTo\n",
      "Compares the values of two fields; useful when requesting a password to be entered twice for\n",
      "confirmation\n",
      "InputRequired Validates that the field contains data before type conversion\n",
      "IPAddress\n",
      "Validates an IPv4 network address\n",
      "Length\n",
      "Validates the length of the string entered\n",
      "MacAddress\n",
      "Validates a MAC address\n",
      "NumberRange\n",
      "Validates that the value entered is within a numeric range\n",
      "Optional\n",
      "Allows empty input in the field, skipping additional validators\n",
      "Regexp\n",
      "Validates the input against a regular expression\n",
      "URL\n",
      "Validates a URL\n",
      "UUID\n",
      "Validates a UUID\n",
      "AnyOf\n",
      "Validates that the input is one of a list of possible values\n",
      "NoneOf\n",
      "Validates that the input is none of a list of possible values\n",
      "46 \n",
      "| \n",
      "Chapter 4: Web Forms\n",
      "\n",
      "HTML Rendering of Forms\n",
      "Form fields are callables that, when invoked from a template, render themselves to\n",
      "HTML. Assuming that the view function passes a NameForm instance to the template\n",
      "as an argument named form, the template can generate a simple HTML form as\n",
      "follows:\n",
      "<form method=\"POST\">\n",
      "    {{ form.hidden_tag() }}\n",
      "    {{ form.name.label }} {{ form.name() }}\n",
      "    {{ form.submit() }}\n",
      "</form>\n",
      "Note that in addition to the name and submit fields, the form has a\n",
      "form.hidden_tag() element. This element defines an extra form field that is hidden,\n",
      "used by Flask-WTF to implement CSRF protection.\n",
      "Of course, the result of rendering a web form in this way is extremely bare. Any key‐\n",
      "word arguments added to the calls that render the fields are converted into HTML\n",
      "attributes for the field—so, for example, you can give the field id or class attributes\n",
      "and then define CSS styles for them:\n",
      "<form method=\"POST\">\n",
      "    {{ form.hidden_tag() }}\n",
      "    {{ form.name.label }} {{ form.name(id='my-text-field') }}\n",
      "    {{ form.submit() }}\n",
      "</form>\n",
      "But even with HTML attributes, the effort required to render a form in this way and\n",
      "make it look good is significant, so it is best to leverage Bootstrap’s own set of form\n",
      "styles whenever possible. The Flask-Bootstrap extension provides a high-level helper\n",
      "function that renders an entire Flask-WTF form using Bootstrap’s predefined form\n",
      "styles, all with a single call. Using Flask-Bootstrap, the previous form can be rendered\n",
      "as follows:\n",
      "{% import \"bootstrap/wtf.html\" as wtf %}\n",
      "{{ wtf.quick_form(form) }}\n",
      "The import directive works in the same way as regular Python scripts do and allows\n",
      "template elements to be imported and used in many templates. The imported boot‐\n",
      "strap/wtf.html file defines helper functions that render Flask-WTF forms using Boot‐\n",
      "strap. The wtf.quick_form() function takes a Flask-WTF form object and renders it\n",
      "using default Bootstrap styles. The complete template for hello.py is shown in\n",
      "Example 4-3.\n",
      "HTML Rendering of Forms \n",
      "| \n",
      "47\n",
      "\n",
      "Example 4-3. templates/index.html: using Flask-WTF and Flask-Bootstrap to render a\n",
      "form\n",
      "{% extends \"base.html\" %}\n",
      "{% import \"bootstrap/wtf.html\" as wtf %}\n",
      "{% block title %}Flasky{% endblock %}\n",
      "{% block page_content %}\n",
      "<div class=\"page-header\">\n",
      "    <h1>Hello, {% if name %}{{ name }}{% else %}Stranger{% endif %}!</h1>\n",
      "</div>\n",
      "{{ wtf.quick_form(form) }}\n",
      "{% endblock %}\n",
      "The content area of the template now has two sections. The first section is a page\n",
      "header that shows a greeting. Here a template conditional is used. Conditionals in\n",
      "Jinja2 have the format {% if condition %}...{% else %}...{% endif %}. If the\n",
      "condition evaluates to True, then what appears between the if and else directives is\n",
      "added to the rendered template. If the condition evaluates to False, then what’s\n",
      "between the else and endif is rendered instead. The purpose of this is to render\n",
      "Hello, {{ name }}! when the name template variable is defined, or the string Hello,\n",
      "Stranger! when it is not. The second section of the content renders the NameForm\n",
      "form using the wtf.quick_form() function.\n",
      "Form Handling in View Functions\n",
      "In the new version of hello.py, the index() view function will have two tasks. First it\n",
      "will render the form, and then it will receive the form data entered by the user.\n",
      "Example 4-4 shows the updated index() view function.\n",
      "Example 4-4. hello.py: handle a web form with GET and POST request methods\n",
      "@app.route('/', methods=['GET', 'POST'])\n",
      "def index():\n",
      "    name = None\n",
      "    form = NameForm()\n",
      "    if form.validate_on_submit():\n",
      "        name = form.name.data\n",
      "        form.name.data = ''\n",
      "    return render_template('index.html', form=form, name=name)\n",
      "The methods argument added to the app.route decorator tells Flask to register the\n",
      "view function as a handler for GET and POST requests in the URL map. When methods\n",
      "is not given, the view function is registered to handle GET requests only.\n",
      "48 \n",
      "| \n",
      "Chapter 4: Web Forms\n",
      "\n",
      "Adding POST to the method list is necessary because form submissions are much\n",
      "more conveniently handled as POST requests. It is possible to submit a form as a GET\n",
      "request, but as GET requests have no body, the data is appended to the URL as a query\n",
      "string and becomes visible in the browser’s address bar. For this and several other rea‐\n",
      "sons, form submissions are almost universally done as POST requests.\n",
      "The local name variable is used to hold the name received from the form when avail‐\n",
      "able; when the name is not known, the variable is initialized to None. The view func‐\n",
      "tion creates an instance of the NameForm class shown previously to represent the form. \n",
      "The validate_on_submit() method of the form returns True when the form was\n",
      "submitted and the data was accepted by all the field validators. In all other cases,\n",
      "validate_on_submit() returns False. The return value of this method effectively\n",
      "serves to determine whether the form needs to be rendered or processed.\n",
      "When a user navigates to the application for the first time, the server will receive a\n",
      "GET request with no form data, so validate_on_submit() will return False. The\n",
      "body of the if statement will be skipped and the request will be handled by rendering\n",
      "the template, which gets the form object and the name variable set to None as argu‐\n",
      "ments. Users will now see the form displayed in the browser.\n",
      "When the form is submitted by the user, the server receives a POST request with the\n",
      "data. The call to validate_on_submit() invokes the DataRequired() validator\n",
      "attached to the name field. If the name is not empty, then the validator accepts it and\n",
      "validate_on_submit() returns True. Now the name entered by the user is accessible\n",
      "as the data attribute of the field. Inside the body of the if statement, this name is\n",
      "assigned to the local name variable and the form field is cleared by setting that data\n",
      "attribute to an empty string, so that the field is blanked when the form is rendered to\n",
      "the page again. The render_template() call in the last line renders the template, but\n",
      "this time the name argument contains the name from the form, so the greeting will be\n",
      "personalized.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 4a to check out this version of the applica‐\n",
      "tion.\n",
      "Figure 4-1 shows how the form looks in the browser window when a user initially\n",
      "enters the site. When the user submits a name, the application responds with a per‐\n",
      "sonalized greeting. The form still appears below it, so a user can submit it multiple\n",
      "times with different names if desired. Figure 4-2 shows the application in this state.\n",
      "Form Handling in View Functions \n",
      "| \n",
      "49\n",
      "\n",
      "Figure 4-1. Flask-WTF web form\n",
      "Figure 4-2. Web form after submission\n",
      "50 \n",
      "| \n",
      "Chapter 4: Web Forms\n",
      "\n",
      "If the user submits the form with an empty name, the DataRequired() validator\n",
      "catches the error, as seen in Figure 4-3. Note how much functionality is being pro‐\n",
      "vided automatically. This is a great example of the power that well-designed exten‐\n",
      "sions like Flask-WTF and Flask-Bootstrap can give to your application.\n",
      "Figure 4-3. Web form after failed validator\n",
      "Redirects and User Sessions\n",
      "The last version of hello.py has a usability problem. If you enter your name and sub‐\n",
      "mit it, and then click the refresh button in your browser, you will likely get an\n",
      "obscure warning that asks for confirmation before submitting the form again. This\n",
      "happens because browsers repeat the last request they sent when they are asked to\n",
      "refresh a page. When the last request sent is a POST request with form data, a refresh\n",
      "would cause a duplicate form submission, which in almost all cases is not the desired\n",
      "action. For that reason, the browser asks for confirmation from the user.\n",
      "Many users do not understand this warning from the browser. Consequently, it is\n",
      "considered good practice for web applications to never leave a POST request as the last\n",
      "request sent by the browser.\n",
      "This is achieved by responding to POST requests with a redirect instead of a normal\n",
      "response. A redirect is a special type of response that contains a URL instead of a\n",
      "string with HTML code. When the browser receives a redirect response, it issues a\n",
      "Redirects and User Sessions \n",
      "| \n",
      "51\n",
      "\n",
      "GET request for the redirect URL, and that is the page that it displays. The page may\n",
      "take a few more milliseconds to load because of the second request that has to be sent\n",
      "to the server, but other than that, the user will not see any difference. Now the last\n",
      "request is a GET, so the refresh command works as expected. This trick is known as\n",
      "the Post/Redirect/Get pattern.\n",
      "But this approach brings a second problem. When the application handles the POST\n",
      "request, it has access to the name entered by the user in form.name.data, but as soon\n",
      "as that request ends the form data is lost. Because the POST request is handled with a\n",
      "redirect, the application needs to store the name so that the redirected request can\n",
      "have it and use it to build the actual response.\n",
      "Applications can “remember” things from one request to the next by storing them in\n",
      "the user session, a private storage that is available to each connected client. The user\n",
      "session was introduced in Chapter 2 as one of the variables associated with the\n",
      "request context. It’s called session and is accessed like a standard Python dictionary.\n",
      "By default, user sessions are stored in client-side cookies that are\n",
      "cryptographically signed using the configured secret key. Any tam‐\n",
      "pering with the cookie content would render the signature invalid,\n",
      "thus invalidating the session.\n",
      "Example 4-5 shows a new version of the index() view function that implements redi‐\n",
      "rects and user sessions.\n",
      "Example 4-5. hello.py: redirects and user sessions\n",
      "from flask import Flask, render_template, session, redirect, url_for\n",
      "@app.route('/', methods=['GET', 'POST'])\n",
      "def index():\n",
      "    form = NameForm()\n",
      "    if form.validate_on_submit():\n",
      "        session['name'] = form.name.data\n",
      "        return redirect(url_for('index'))\n",
      "    return render_template('index.html', form=form, name=session.get('name'))\n",
      "In the previous version of the application, a local name variable was used to store the\n",
      "name entered by the user in the form. That variable is now placed in the user session\n",
      "as session['name'] so that it is remembered beyond the request.\n",
      "52 \n",
      "| \n",
      "Chapter 4: Web Forms\n",
      "\n",
      "Requests that come with valid form data will now end with a call to redirect(), a\n",
      "Flask helper function that generates the HTTP redirect response. The redirect()\n",
      "function takes the URL to redirect to as an argument. The redirect URL used in this\n",
      "case is the root URL, so the response could have been written more concisely as\n",
      "redirect('/'), but instead Flask’s URL generator function url_for(), introduced in\n",
      "Chapter 3, is used.\n",
      "The first and only required argument to url_for() is the endpoint name, the internal\n",
      "name each route has. By default, the endpoint of a route is the name of the view func‐\n",
      "tion attached to it. In this example, the view function that handles the root URL is\n",
      "index(), so the name given to url_for() is index.\n",
      "The last change is in the render_template() function, which now obtains the name\n",
      "argument directly from the session using session.get('name'). As with regular dic‐\n",
      "tionaries, using get() to request a dictionary key avoids an exception for keys that\n",
      "aren’t found. The get() method returns a default value of None for a missing key.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 4b to check out this version of the applica‐\n",
      "tion.\n",
      "With this version of the application, you can see that refreshing the page in your\n",
      "browser always results in the expected behavior.\n",
      "Message Flashing\n",
      "Sometimes it is useful to give the user a status update after a request is completed.\n",
      "This could be a confirmation message, a warning, or an error. A typical example is\n",
      "when you submit a login form to a website with a mistake and the server responds by\n",
      "rendering the login form again with a message above it that informs you that your\n",
      "username or password is invalid.\n",
      "Flask includes this functionality as a core feature. Example 4-6 shows how the\n",
      "flash() function can be used for this purpose.\n",
      "Message Flashing \n",
      "| \n",
      "53\n",
      "\n",
      "Example 4-6. hello.py: flashed messages\n",
      "from flask import Flask, render_template, session, redirect, url_for, flash\n",
      "@app.route('/', methods=['GET', 'POST'])\n",
      "def index():\n",
      "    form = NameForm()\n",
      "    if form.validate_on_submit():\n",
      "        old_name = session.get('name')\n",
      "        if old_name is not None and old_name != form.name.data:\n",
      "            flash('Looks like you have changed your name!')\n",
      "        session['name'] = form.name.data\n",
      "        return redirect(url_for('index'))\n",
      "    return render_template('index.html',\n",
      "        form = form, name = session.get('name'))\n",
      "In this example, each time a name is submitted it is compared against the name\n",
      "stored in the user session, which will have been put there during a previous submis‐\n",
      "sion of the same form. If the two names are different, the flash() function is invoked\n",
      "with a message to be displayed on the next response sent back to the client.\n",
      "Calling flash() is not enough to get messages displayed; the templates used by the\n",
      "application need to render these messages. The best place to render flashed messages\n",
      "is the base template, because that will enable these messages in all pages. Flask makes\n",
      "a get_flashed_messages() function available to templates to retrieve the messages\n",
      "and render them, as shown in Example 4-7.\n",
      "Example 4-7. templates/base.html: rendering of flashed messages\n",
      "{% block content %}\n",
      "<div class=\"container\">\n",
      "    {% for message in get_flashed_messages() %}\n",
      "    <div class=\"alert alert-warning\">\n",
      "        <button type=\"button\" class=\"close\" data-dismiss=\"alert\">&times;</button>\n",
      "        {{ message }}\n",
      "    </div>\n",
      "    {% endfor %}\n",
      "    {% block page_content %}{% endblock %}\n",
      "</div>\n",
      "{% endblock %}\n",
      "In this example, messages are rendered using Bootstrap’s alert CSS styles for warning\n",
      "messages (one is shown in Figure 4-4).\n",
      "54 \n",
      "| \n",
      "Chapter 4: Web Forms\n",
      "\n",
      "Figure 4-4. Flashed message\n",
      "A loop is used because there could be multiple messages queued for display, one for\n",
      "each time flash() was called in the previous request cycle. Messages that are\n",
      "retrieved from get_flashed_messages() will not be returned the next time this func‐\n",
      "tion is called, so flashed messages appear only once and are then discarded.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 4c to check out this version of the applica‐\n",
      "tion.\n",
      "Being able to accept data from the user through web forms is a feature required by\n",
      "most applications, and so is the ability to store that data in permanent storage. Using\n",
      "databases with Flask is the topic of the next chapter.\n",
      "Message Flashing \n",
      "| \n",
      "55\n",
      "\n",
      "\n",
      "CHAPTER 5\n",
      "Databases\n",
      "A database stores application data in an organized way. The application then issues\n",
      "queries to retrieve specific portions of the data as they are needed. The most com‐\n",
      "monly used databases for web applications are those based on the relational model,\n",
      "also called SQL databases in reference to the Structured Query Language they use.\n",
      "But in recent years document-oriented and key-value databases, informally known\n",
      "together as NoSQL databases, have become popular alternatives.\n",
      "SQL Databases\n",
      "Relational databases store data in tables, which model the different entities in the\n",
      "application’s domain. For example, a database for an order management application\n",
      "will likely have customers, products, and orders tables.\n",
      "A table has a fixed number of columns and a variable number of rows. The columns\n",
      "define the data attributes of the entity represented by the table. For example, a\n",
      "customers table will have columns such as name, address, phone, and so on. Each\n",
      "row in a table defines an actual data element that assigns values to some or all the\n",
      "columns.\n",
      "Tables have a special column called the primary key, which holds a unique identifier\n",
      "for each row stored in the table. Tables can also have columns called foreign keys,\n",
      "which reference the primary key of a row in the same or another table. These links\n",
      "between rows are called relationships and are the foundation of the relational database\n",
      "model.\n",
      "Figure 5-1 shows a diagram of a simple database with two tables that store users and\n",
      "user roles. The line that connects the two tables represents a relationship between the\n",
      "tables.\n",
      "57\n",
      "\n",
      "Figure 5-1. Relational database example\n",
      "This graphical style of representing the structure of a database is called an entity-\n",
      "relationship diagram. In this representation, boxes represent database tables, showing\n",
      "lists of the table’s attributes or columns. The roles table stores the list of all possible\n",
      "user roles, each identified by a unique id value—the table’s primary key. The users\n",
      "table contains the list of users, each with its own unique id as well. Besides the id\n",
      "primary keys, the roles table has a name column and the users table has username\n",
      "and password columns.\n",
      "The role_id column in the users table is a foreign key. The line that connects the\n",
      "roles.id and users.role_id columns represents a relationship between the two\n",
      "tables. The symbols attached to the line at each end indicate the cardinality of the\n",
      "relationship. On the roles.id side, the line is shown to have a “one,” while on the\n",
      "users.role_id side a “many” is represented. This depicts a one-to-many relationship,\n",
      "indicating that each row from the roles table can be associated with many rows from\n",
      "the users table.\n",
      "As seen in the example, relational databases store data efficiently and avoid duplica‐\n",
      "tion. Renaming a user role in this database is simple because role names exist in a\n",
      "single place. Immediately after a role name is changed in the roles table, all users\n",
      "that have a role_id that references the changed role will see the update.\n",
      "On the other hand, having the data split into multiple tables can be a complication.\n",
      "Producing a listing of users with their roles presents a small problem, because users\n",
      "and user roles need to be read from two tables and joined before they can be presen‐\n",
      "ted together. Relational database engines provide the support to perform join opera‐\n",
      "tions between tables when necessary.\n",
      "NoSQL Databases\n",
      "Databases that do not follow the relational model described in the previous section\n",
      "are collectively referred to as NoSQL databases. One common organization for\n",
      "NoSQL databases uses collections instead of tables and documents instead of records.\n",
      "NoSQL databases are designed in a way that makes joins difficult, so most of them do\n",
      "not support this operation at all. For a NoSQL database structured as in Figure 5-1,\n",
      "58 \n",
      "| \n",
      "Chapter 5: Databases\n",
      "\n",
      "listing the users with their roles requires the application itself to perform the join\n",
      "operation by reading the role_id field of each user and then searching the roles\n",
      "table for it.\n",
      "A more appropriate design for a NoSQL database is shown in Figure 5-2. This is the\n",
      "result of applying an operation called denormalization, which reduces the number of\n",
      "tables at the expense of data duplication.\n",
      "Figure 5-2. NoSQL database example\n",
      "A database with this structure has the role name explicitly stored with each user.\n",
      "Renaming a role can then turn out to be an expensive operation that may require\n",
      "updating a large number of documents.\n",
      "But it isn’t all bad news with NoSQL databases. Having the data duplicated allows for\n",
      "faster querying. Listing users and their roles is straightforward because no joins are\n",
      "needed.\n",
      "SQL or NoSQL?\n",
      "SQL databases excel at storing structured data in an efficient and compact form.\n",
      "These databases go to great lengths to preserve consistency, even in the face of power\n",
      "failures or hardware malfunctions. The paradigm that allows relational databases to\n",
      "reach this high level of reliability is called ACID, which stands for Atomicity, Consis‐\n",
      "tency, Isolation, and Durability. NoSQL databases relax some of the ACID require‐\n",
      "ments and as a result can sometimes get a performance edge.\n",
      "A full analysis and comparison of database types is outside the scope of this book. For\n",
      "small to medium-sized applications, both SQL and NoSQL databases are perfectly\n",
      "capable and have practically equivalent performance.\n",
      "Python Database Frameworks\n",
      "Python has packages for most database engines, both open source and commercial.\n",
      "Flask puts no restrictions on what database packages can be used, so you can work\n",
      "with MySQL, Postgres, SQLite, Redis, MongoDB, CouchDB, or DynamoDB if any of\n",
      "these is your favorite.\n",
      "SQL or NoSQL? \n",
      "| \n",
      "59\n",
      "\n",
      "As if those weren’t enough choices, there are also a number of database abstraction\n",
      "layer packages, such as SQLAlchemy or MongoEngine, that allow you to work at a\n",
      "higher level with regular Python objects instead of database entities such as tables,\n",
      "documents, or query languages.\n",
      "There are a number of factors to evaluate when choosing a database framework:\n",
      "Ease of use\n",
      "When comparing straight database engines to database abstraction layers, the\n",
      "second group clearly wins. Abstraction layers, also called object-relational map‐\n",
      "pers (ORMs) or object-document mappers (ODMs), provide transparent conver‐\n",
      "sion of high-level object-oriented operations into low-level database instructions.\n",
      "Performance\n",
      "The conversions that ORMs and ODMs have to do to translate from the object\n",
      "domain into the database domain have an overhead. In most cases, the perfor‐\n",
      "mance penalty is negligible, but it may not always be. In general, the productivity\n",
      "gain obtained with ORMs and ODMs far outweighs a minimal performance deg‐\n",
      "radation, so this isn’t a valid argument to drop ORMs and ODMs completely.\n",
      "What makes sense is to choose a database abstraction layer that provides optional\n",
      "access to the underlying database in case specific operations need to be optimized\n",
      "by implementing them directly as native database instructions.\n",
      "Portability\n",
      "The database choices available on your development and production platforms\n",
      "must be considered. For example, if you plan to host your application on a cloud\n",
      "platform, then you should find out what database choices this service offers.\n",
      "Another portability aspect applies to ORMs and ODMs. Although some of these\n",
      "frameworks provide an abstraction layer for a single database engine, others\n",
      "abstract even higher and provide a choice of database engines—all accessible\n",
      "with the same object-oriented interface. The best example of this is the SQL‐\n",
      "Alchemy ORM, which supports a list of relational database engines including the\n",
      "popular MySQL, Postgres, and SQLite.\n",
      "Flask integration\n",
      "Choosing a framework that has integration with Flask is not absolutely required,\n",
      "but it will save you from having to write the integration code yourself. Flask inte‐\n",
      "gration could simplify configuration and operation, so using a package specifi‐\n",
      "cally designed as a Flask extension should be preferred.\n",
      "Based on these goals, the chosen database framework for the examples in this book\n",
      "will be Flask-SQLAlchemy, the Flask extension wrapper for SQLAlchemy.\n",
      "60 \n",
      "| \n",
      "Chapter 5: Databases\n",
      "\n",
      "Database Management with Flask-SQLAlchemy\n",
      "Flask-SQLAlchemy is a Flask extension that simplifies the use of SQLAlchemy inside\n",
      "Flask applications. SQLAlchemy is a powerful relational database framework that\n",
      "supports several database backends. It offers a high-level ORM and low-level access\n",
      "to the database’s native SQL functionality.\n",
      "Like most other extensions, Flask-SQLAlchemy is installed with pip:\n",
      "(venv) $ pip install flask-sqlalchemy\n",
      "In Flask-SQLAlchemy, a database is specified as a URL. Table 5-1 lists the format of\n",
      "the URLs for the three most popular database engines.\n",
      "Table 5-1. Flask-SQLAlchemy database URLs\n",
      "Database engine\n",
      "URL\n",
      "MySQL\n",
      "mysql://username:password@hostname/database\n",
      "Postgres\n",
      "postgresql://username:password@hostname/database\n",
      "SQLite (Linux, macOS) sqlite:////absolute/path/to/database\n",
      "SQLite (Windows)\n",
      "sqlite:///c:/absolute/path/to/database\n",
      "In these URLs, hostname refers to the server that hosts the database service, which\n",
      "could be localhost or a remote server. Database servers can host several databases, so\n",
      "database indicates the name of the database to use. For databases that need authenti‐\n",
      "cation, username and password are the database user credentials.\n",
      "SQLite databases do not have a server, so hostname, username, and\n",
      "password are omitted and database is the filename on disk for the\n",
      "database.\n",
      "The URL of the application database must be configured as the key\n",
      "SQLALCHEMY_DATABASE_URI in the Flask configuration object. The Flask-SQLAlchemy\n",
      "documentation also suggests setting key SQLALCHEMY_TRACK_MODIFICATIONS to False\n",
      "to use less memory unless signals for object changes are needed. Consult the Flask-\n",
      "SQLAlchemy documentation for information on other configuration options.\n",
      "Example 5-1 shows how to initialize and configure a simple SQLite database.\n",
      "Example 5-1. hello.py: database configuration\n",
      "import os\n",
      "from flask_sqlalchemy import SQLAlchemy\n",
      "basedir = os.path.abspath(os.path.dirname(__file__))\n",
      "Database Management with Flask-SQLAlchemy \n",
      "| \n",
      "61\n",
      "\n",
      "app = Flask(__name__)\n",
      "app.config['SQLALCHEMY_DATABASE_URI'] =\\\n",
      "    'sqlite:///' + os.path.join(basedir, 'data.sqlite')\n",
      "app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False\n",
      "db = SQLAlchemy(app)\n",
      "The db object instantiated from the class SQLAlchemy represents the database and\n",
      "provides access to all the functionality of Flask-SQLAlchemy.\n",
      "Model Definition\n",
      "The term model is used when referring to the persistent entities used by the applica‐\n",
      "tion. In the context of an ORM, a model is typically a Python class with attributes that\n",
      "match the columns of a corresponding database table.\n",
      "The database instance from Flask-SQLAlchemy provides a base class for models as\n",
      "well as a set of helper classes and functions that are used to define their structure. The\n",
      "roles and users tables from Figure 5-1 can be defined as the models Role and User\n",
      "as shown in Example 5-2.\n",
      "Example 5-2. hello.py: Role and User model definition\n",
      "class Role(db.Model):\n",
      "    __tablename__ = 'roles'\n",
      "    id = db.Column(db.Integer, primary_key=True)\n",
      "    name = db.Column(db.String(64), unique=True)\n",
      "    def __repr__(self):\n",
      "        return '<Role %r>' % self.name\n",
      "class User(db.Model):\n",
      "    __tablename__ = 'users'\n",
      "    id = db.Column(db.Integer, primary_key=True)\n",
      "    username = db.Column(db.String(64), unique=True, index=True)\n",
      "    def __repr__(self):\n",
      "        return '<User %r>' % self.username\n",
      "The __tablename__ class variable defines the name of the table in the database. Flask-\n",
      "SQLAlchemy assigns a default table name if __tablename__ is omitted, but those\n",
      "default names do not follow the popular convention of using plurals for table names,\n",
      "so it is best to name tables explicitly. The remaining class variables are the attributes\n",
      "of the model, defined as instances of the db.Column class.\n",
      "62 \n",
      "| \n",
      "Chapter 5: Databases\n",
      "\n",
      "The first argument given to the db.Column constructor is the type of the database col‐\n",
      "umn and model attribute. Table 5-2 lists some of the column types that are available,\n",
      "along with the Python types used in the model.\n",
      "Table 5-2. Most common SQLAlchemy column types\n",
      "Type name\n",
      "Python type\n",
      "Description\n",
      "Integer\n",
      "int\n",
      "Regular integer, typically 32 bits\n",
      "SmallInteger\n",
      "int\n",
      "Short-range integer, typically 16 bits\n",
      "BigInteger\n",
      "int or long\n",
      "Unlimited precision integer\n",
      "Float\n",
      "float\n",
      "Floating-point number\n",
      "Numeric\n",
      "decimal.Decimal\n",
      "Fixed-point number\n",
      "String\n",
      "str\n",
      "Variable-length string\n",
      "Text\n",
      "str\n",
      "Variable-length string, optimized for large or unbounded length\n",
      "Unicode\n",
      "unicode\n",
      "Variable-length Unicode string\n",
      "UnicodeText\n",
      "unicode\n",
      "Variable-length Unicode string, optimized for large or unbounded length\n",
      "Boolean\n",
      "bool\n",
      "Boolean value\n",
      "Date\n",
      "datetime.date\n",
      "Date value\n",
      "Time\n",
      "datetime.time\n",
      "Time value\n",
      "DateTime\n",
      "datetime.datetime\n",
      "Date and time value\n",
      "Interval\n",
      "datetime.timedelta Time interval\n",
      "Enum\n",
      "str\n",
      "List of string values\n",
      "PickleType\n",
      "Any Python object\n",
      "Automatic Pickle serialization\n",
      "LargeBinary\n",
      "str\n",
      "Binary blob\n",
      "The remaining arguments to db.Column specify configuration options for each\n",
      "attribute. Table 5-3 lists some of the options available.\n",
      "Table 5-3. Most common SQLAlchemy column options\n",
      "Option name\n",
      "Description\n",
      "primary_key\n",
      "If set to True, the column is the table’s primary key.\n",
      "unique\n",
      "If set to True, do not allow duplicate values for this column.\n",
      "index\n",
      "If set to True, create an index for this column, so that queries are more efficient.\n",
      "nullable\n",
      "If set to True, allow empty values for this column. If set to False, the column will not allow null values.\n",
      "default\n",
      "Define a default value for the column.\n",
      "Model Definition \n",
      "| \n",
      "63\n",
      "\n",
      "Flask-SQLAlchemy requires all models to define a primary key col‐\n",
      "umn, which is commonly named id.\n",
      "Although it’s not strictly necessary, the two models include a __repr__() method to\n",
      "give them a readable string representation that can be used for debugging and testing\n",
      "purposes.\n",
      "Relationships\n",
      "Relational databases establish connections between rows in different tables through\n",
      "the use of relationships. The relational diagram in Figure 5-1 expresses a simple rela‐\n",
      "tionship between users and their roles. This is a one-to-many relationship from roles\n",
      "to users, because one role can belong to many users, but each user can have only one\n",
      "role.\n",
      "Example 5-3 shows how the one-to-many relationship in Figure 5-1 is represented in\n",
      "the model classes.\n",
      "Example 5-3. hello.py: relationships in the database models\n",
      "class Role(db.Model):\n",
      "    # ...\n",
      "    users = db.relationship('User', backref='role')\n",
      "class User(db.Model):\n",
      "    # ...\n",
      "    role_id = db.Column(db.Integer, db.ForeignKey('roles.id'))\n",
      "As seen in Figure 5-1, a relationship connects two rows through the use of a foreign\n",
      "key. The role_id column added to the User model is defined as a foreign key, and\n",
      "that establishes the relationship. The 'roles.id' argument to db.ForeignKey()\n",
      "specifies that the column should be interpreted as having id values from rows in the\n",
      "roles table.\n",
      "The users attribute added to the model Role represents the object-oriented view of\n",
      "the relationship, as seen from the “one” side. Given an instance of class Role, the\n",
      "users attribute will return the list of users associated with that role (i.e., the “many”\n",
      "side). The first argument to db.relationship() indicates what model is on the other\n",
      "side of the relationship. The model class can be provided as a string if the class is\n",
      "defined later in the module.\n",
      "The backref argument to db.relationship() defines the reverse direction of the\n",
      "relationship, by adding a role attribute to the User model. This attribute can be used\n",
      "64 \n",
      "| \n",
      "Chapter 5: Databases\n",
      "\n",
      "on any instance of User instead of the role_id foreign key to access the Role model\n",
      "as an object.\n",
      "In most cases db.relationship() can locate the relationship’s foreign key on its own,\n",
      "but sometimes it cannot determine what column to use as a foreign key. For example,\n",
      "if the User model had two or more columns defined as Role foreign keys, then\n",
      "SQLAlchemy would not know which one of the two to use. Whenever the foreign key\n",
      "configuration is ambiguous, additional arguments to db.relationship() need to be\n",
      "given. Table 5-4 lists some of the common configuration options that can be used to\n",
      "define a relationship.\n",
      "Table 5-4. Common SQLAlchemy relationship options\n",
      "Option name\n",
      "Description\n",
      "backref\n",
      "Add a back reference in the other model in the relationship.\n",
      "primaryjoin\n",
      "Specify the join condition between the two models explicitly. This is necessary only for ambiguous\n",
      "relationships.\n",
      "lazy\n",
      "Specify how the related items are to be loaded. Possible values are select (items are loaded on\n",
      "demand the first time they are accessed), immediate (items are loaded when the source object is\n",
      "loaded), joined (items are loaded immediately, but as a join), subquery (items are loaded\n",
      "immediately, but as a subquery), noload (items are never loaded), and dynamic (instead of loading\n",
      "the items, the query that can load them is given).\n",
      "uselist\n",
      "If set to False, use a scalar instead of a list.\n",
      "order_by\n",
      "Specify the ordering used for the items in the relationship.\n",
      "secondary\n",
      "Specify the name of the association table to use in many-to-many relationships.\n",
      "secondaryjoin Specify the secondary join condition for many-to-many relationships when SQLAlchemy cannot\n",
      "determine it on its own.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 5a to check out this version of the applica‐\n",
      "tion.\n",
      "There are other relationship types besides one-to-many. The one-to-one relationship\n",
      "can be expressed the same way as one-to-many, as described earlier, but with the\n",
      "uselist option set to False within the db.relationship() definition so that the\n",
      "“many” side becomes a “one” side. The many-to-one relationship can also be\n",
      "expressed as a one-to-many if the tables are reversed, or it can be expressed with the\n",
      "foreign key and the db.relationship() definition both on the “many” side. The\n",
      "most complex relationship type, many-to-many, requires an additional table called an\n",
      "association or junction table. You will learn about many-to-many relationships in\n",
      "Chapter 12.\n",
      "Relationships \n",
      "| \n",
      "65\n",
      "\n",
      "Database Operations\n",
      "The models are now fully configured according to the database diagram in Figure 5-1\n",
      "and are ready to be used. The best way to learn how to work with these models is in a\n",
      "Python shell. The following sections will walk you through the most common data‐\n",
      "base operations in a shell started with the flask shell command. Before you use\n",
      "this command, make sure the FLASK_APP environment variable is set to hello.py, as\n",
      "shown in Chapter 2.\n",
      "Creating the Tables\n",
      "The very first thing to do is to instruct Flask-SQLAlchemy to create a database based\n",
      "on the model classes. The db.create_all() function locates all the subclasses of\n",
      "db.Model and creates corresponding tables in the database for them:\n",
      "(venv) $ flask shell\n",
      ">>> from hello import db\n",
      ">>> db.create_all()\n",
      "If you check the application directory, you will now see a new file there called\n",
      "data.sqlite, the name that was given to the SQLite database in the configuration. The\n",
      "db.create_all() function will not re-create or update a database table if it already\n",
      "exists in the database. This can be inconvenient when the models are modified and\n",
      "the changes need to be applied to an existing database. The brute-force solution to\n",
      "update existing database tables to a different schema is to remove the old tables first:\n",
      ">>> db.drop_all()\n",
      ">>> db.create_all()\n",
      "Unfortunately, this method has the undesired side effect of destroying all the data in\n",
      "the old database. A better solution to the problem of updating databases is presented\n",
      "near the end of the chapter.\n",
      "Inserting Rows\n",
      "The following example creates a few roles and users:\n",
      ">>> from hello import Role, User\n",
      ">>> admin_role = Role(name='Admin')\n",
      ">>> mod_role = Role(name='Moderator')\n",
      ">>> user_role = Role(name='User')\n",
      ">>> user_john = User(username='john', role=admin_role)\n",
      ">>> user_susan = User(username='susan', role=user_role)\n",
      ">>> user_david = User(username='david', role=user_role)\n",
      "The constructors for models accept initial values for the model attributes as keyword\n",
      "arguments. Note that the role attribute can be used, even though it is not a real data‐\n",
      "base column but a high-level representation of the one-to-many relationship. The id\n",
      "66 \n",
      "| \n",
      "Chapter 5: Databases\n",
      "\n",
      "attribute of these new objects is not set explicitly: the primary keys in many databases\n",
      "are managed by the database itself. The objects exist only on the Python side so far;\n",
      "they have not been written to the database yet. Because of that, their id values have\n",
      "not yet been assigned:\n",
      ">>> print(admin_role.id)\n",
      "None\n",
      ">>> print(mod_role.id)\n",
      "None\n",
      ">>> print(user_role.id)\n",
      "None\n",
      "Changes to the database are managed through a database session, which Flask-\n",
      "SQLAlchemy provides as db.session. To prepare objects to be written to the data‐\n",
      "base, they must be added to the session:\n",
      ">>> db.session.add(admin_role)\n",
      ">>> db.session.add(mod_role)\n",
      ">>> db.session.add(user_role)\n",
      ">>> db.session.add(user_john)\n",
      ">>> db.session.add(user_susan)\n",
      ">>> db.session.add(user_david)\n",
      "Or, more concisely:\n",
      ">>> db.session.add_all([admin_role, mod_role, user_role,\n",
      "...     user_john, user_susan, user_david])\n",
      "To write the objects to the database, the session needs to be committed by calling its\n",
      "commit() method:\n",
      ">>> db.session.commit()\n",
      "Check the id attributes again after having the data committed to see that they are\n",
      "now set:\n",
      ">>> print(admin_role.id)\n",
      "1\n",
      ">>> print(mod_role.id)\n",
      "2\n",
      ">>> print(user_role.id)\n",
      "3\n",
      "The db.session database session is not related to the Flask\n",
      "session object discussed in Chapter 4. Database sessions are also\n",
      "called transactions.\n",
      "Database sessions are extremely useful in keeping the database consistent. The com‐\n",
      "mit operation writes all the objects that were added to the session atomically. If an\n",
      "Database Operations \n",
      "| \n",
      "67\n",
      "\n",
      "error occurs while the session is being written, the whole session is discarded. If you\n",
      "always commit related changes together in a session, you are guaranteed to avoid\n",
      "database inconsistencies due to partial updates.\n",
      "A \n",
      "database \n",
      "session \n",
      "can \n",
      "also \n",
      "be \n",
      "rolled \n",
      "back. \n",
      "If\n",
      "db.session.rollback() is called, any objects that were added to\n",
      "the database session are restored to the state they have in the data‐\n",
      "base.\n",
      "Modifying Rows\n",
      "The add() method of the database session can also be used to update models. Con‐\n",
      "tinuing in the same shell session, the following example renames the \"Admin\" role to\n",
      "\"Administrator\":\n",
      ">>> admin_role.name = 'Administrator'\n",
      ">>> db.session.add(admin_role)\n",
      ">>> db.session.commit()\n",
      "Deleting Rows\n",
      "The database session also has a delete() method. The following example deletes the\n",
      "\"Moderator\" role from the database:\n",
      ">>> db.session.delete(mod_role)\n",
      ">>> db.session.commit()\n",
      "Note that deletions, like insertions and updates, are executed only when the database\n",
      "session is committed.\n",
      "Querying Rows\n",
      "Flask-SQLAlchemy makes a query object available in each model class. The most\n",
      "basic query for a model is triggered with the all() method, which returns the entire\n",
      "contents of the corresponding table:\n",
      ">>> Role.query.all()\n",
      "[<Role 'Administrator'>, <Role 'User'>]\n",
      ">>> User.query.all()\n",
      "[<User 'john'>, <User 'susan'>, <User 'david'>]\n",
      "A query object can be configured to issue more specific database searches through\n",
      "the use of filters. The following example finds all the users that were assigned the\n",
      "\"User\" role:\n",
      ">>> User.query.filter_by(role=user_role).all()\n",
      "[<User 'susan'>, <User 'david'>]\n",
      "68 \n",
      "| \n",
      "Chapter 5: Databases\n",
      "\n",
      "It is also possible to inspect the native SQL query that SQLAlchemy generates for a\n",
      "given query by converting the query object to a string:\n",
      ">>> str(User.query.filter_by(role=user_role))\n",
      "'SELECT users.id AS users_id, users.username AS users_username,\n",
      "users.role_id AS users_role_id \\nFROM users \\nWHERE :param_1 = users.role_id'\n",
      "If you exit the shell session, the objects created in the previous example will cease to\n",
      "exist as Python objects but will continue to exist as rows in their respective database\n",
      "tables. If you then start a brand-new shell session, you have to re-create the Python\n",
      "objects from their database rows. The following example issues a query that loads the\n",
      "user role with name \"User\":\n",
      ">>> user_role = Role.query.filter_by(name='User').first()\n",
      "Note how in this case, the query was issued with the first() method instead of\n",
      "all(). While all() returns all the results of the query as a list, first() returns only\n",
      "the first result or None if there are no results, so it is a convenient method to use for\n",
      "queries that are known to return one result at the most.\n",
      "Filters such as filter_by() are invoked on a query object and return a new refined\n",
      "query. Multiple filters can be called in sequence until the query is configured as\n",
      "needed.\n",
      "Table 5-5 shows some of the most common filters available to queries. The complete\n",
      "list is in the SQLAlchemy documentation.\n",
      "Table 5-5. Common SQLAlchemy query filters\n",
      "Option\n",
      "Description\n",
      "filter()\n",
      "Returns a new query that adds an additional filter to the original query\n",
      "filter_by()\n",
      "Returns a new query that adds an additional equality filter to the original query\n",
      "limit()\n",
      "Returns a new query that limits the number of results of the original query to the given number\n",
      "offset()\n",
      "Returns a new query that applies an offset into the list of results of the original query\n",
      "order_by()\n",
      "Returns a new query that sorts the results of the original query according to the given criteria\n",
      "group_by()\n",
      "Returns a new query that groups the results of the original query according to the given criteria\n",
      "After the desired filters have been applied to the query, a call to all() will cause the\n",
      "query to execute and return the results as a list—but there are other ways to trigger\n",
      "the execution of a query besides all(). Table 5-6 shows other query execution meth‐\n",
      "ods.\n",
      "Database Operations \n",
      "| \n",
      "69\n",
      "\n",
      "Table 5-6. Most common SQLAlchemy query executors\n",
      "Option\n",
      "Description\n",
      "all()\n",
      "Returns all the results of a query as a list\n",
      "first()\n",
      "Returns the first result of a query, or None if there are no results\n",
      "first_or_404() Returns the first result of a query, or aborts the request and sends a 404 error as the response if there\n",
      "are no results\n",
      "get()\n",
      "Returns the row that matches the given primary key, or None if no matching row is found\n",
      "get_or_404()\n",
      "Returns the row that matches the given primary key or, if the key is not found, aborts the request and\n",
      "sends a 404 error as the response\n",
      "count()\n",
      "Returns the result count of the query\n",
      "paginate()\n",
      "Returns a Pagination object that contains the specified range of results\n",
      "Relationships work similarly to queries. The following example queries the one-to-\n",
      "many relationship between roles and users from both ends:\n",
      ">>> users = user_role.users\n",
      ">>> users\n",
      "[<User 'susan'>, <User 'david'>]\n",
      ">>> users[0].role\n",
      "<Role 'User'>\n",
      "The user_role.users query here has a small problem. The implicit query that runs\n",
      "when the user_role.users expression is issued internally calls all() to return the\n",
      "list of users. Because the query object is hidden, it is not possible to refine it with\n",
      "additional query filters. In this particular example, it may have been useful to request\n",
      "that the user list be returned in alphabetical order. In Example 5-4, the configuration\n",
      "of the relationship is modified with a lazy='dynamic' argument to request that the\n",
      "query is not automatically executed.\n",
      "Example 5-4. hello.py: dynamic database relationships\n",
      "class Role(db.Model):\n",
      "    # ...\n",
      "    users = db.relationship('User', backref='role', lazy='dynamic')\n",
      "    # ...\n",
      "With the relationship configured in this way, user_role.users returns a query that\n",
      "hasn’t executed yet, so filters can be added to it:\n",
      ">>> user_role.users.order_by(User.username).all()\n",
      "[<User 'david'>, <User 'susan'>]\n",
      ">>> user_role.users.count()\n",
      "2\n",
      "70 \n",
      "| \n",
      "Chapter 5: Databases\n",
      "\n",
      "Database Use in View Functions\n",
      "The database operations described in the previous sections can be used directly inside\n",
      "view functions. Example 5-5 shows a new version of the home page route that records\n",
      "names entered by users in the database.\n",
      "Example 5-5. hello.py: database use in view functions\n",
      "@app.route('/', methods=['GET', 'POST'])\n",
      "def index():\n",
      "    form = NameForm()\n",
      "    if form.validate_on_submit():\n",
      "        user = User.query.filter_by(username=form.name.data).first()\n",
      "        if user is None:\n",
      "            user = User(username=form.name.data)\n",
      "            db.session.add(user)\n",
      "            db.session.commit()\n",
      "            session['known'] = False\n",
      "        else:\n",
      "            session['known'] = True\n",
      "        session['name'] = form.name.data\n",
      "        form.name.data = ''\n",
      "        return redirect(url_for('index'))\n",
      "    return render_template('index.html',\n",
      "        form=form, name=session.get('name'),\n",
      "        known=session.get('known', False))\n",
      "In this modified version of the application, each time a name is submitted the appli‐\n",
      "cation checks for it in the database using the filter_by() query filter. A known vari‐\n",
      "able is written to the user session so that after the redirect the information can be sent\n",
      "to the template, where it is used to customize the greeting. Note that for the applica‐\n",
      "tion to work, the database tables must be created in a Python shell as shown earlier.\n",
      "The new version of the associated template is shown in Example 5-6. This template\n",
      "uses the known argument to add a second line to the greeting that is different for\n",
      "known and new users.\n",
      "Example 5-6. templates/index.html: customized greeting in template\n",
      "{% extends \"base.html\" %}\n",
      "{% import \"bootstrap/wtf.html\" as wtf %}\n",
      "{% block title %}Flasky{% endblock %}\n",
      "{% block page_content %}\n",
      "<div class=\"page-header\">\n",
      "    <h1>Hello, {% if name %}{{ name }}{% else %}Stranger{% endif %}!</h1>\n",
      "    {% if not known %}\n",
      "Database Use in View Functions \n",
      "| \n",
      "71\n",
      "\n",
      "    <p>Pleased to meet you!</p>\n",
      "    {% else %}\n",
      "    <p>Happy to see you again!</p>\n",
      "    {% endif %}\n",
      "</div>\n",
      "{{ wtf.quick_form(form) }}\n",
      "{% endblock %}\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 5b to check out this version of the applica‐\n",
      "tion.\n",
      "Integration with the Python Shell\n",
      "Having to import the database instance and the models each time a shell session is\n",
      "started is tedious work. To avoid having to constantly repeat these steps, the flask\n",
      "shell command can be configured to automatically import these objects.\n",
      "To add objects to the import list, a shell context processor must be created and regis‐\n",
      "tered with the app.shell_context_processor decorator. This is shown in\n",
      "Example 5-7.\n",
      "Example 5-7. hello.py: adding a shell context\n",
      "@app.shell_context_processor\n",
      "def make_shell_context():\n",
      "    return dict(db=db, User=User, Role=Role)\n",
      "The shell context processor function returns a dictionary that includes the database\n",
      "instance and the models. The flask shell command will import these items auto‐\n",
      "matically into the shell, in addition to app, which is imported by default:\n",
      "$ flask shell\n",
      ">>> app\n",
      "<Flask 'hello'>\n",
      ">>> db\n",
      "<SQLAlchemy engine='sqlite:////home/flask/flasky/data.sqlite'>\n",
      ">>> User\n",
      "<class 'hello.User'>\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 5c to check out this version of the applica‐\n",
      "tion.\n",
      "72 \n",
      "| \n",
      "Chapter 5: Databases\n",
      "\n",
      "Database Migrations with Flask-Migrate\n",
      "As you make progress developing an application, you will find that your database\n",
      "models need to change, and when that happens the database needs to be updated as\n",
      "well. Flask-SQLAlchemy creates database tables from models only when they do not\n",
      "exist already, so the only way to make it update tables is by destroying the old tables\n",
      "first—but of course, this causes all the data in the database to be lost.\n",
      "A better solution is to use a database migration framework. In the same way source\n",
      "code version control tools keep track of changes to source code files, a database\n",
      "migration framework keeps track of changes to a database schema, allowing incre‐\n",
      "mental changes to be applied.\n",
      "The developer of SQLAlchemy has written a migration framework called Alembic,\n",
      "but instead of using Alembic directly, Flask applications can use the Flask-Migrate\n",
      "extension, a lightweight Alembic wrapper that integrates it with the flask command.\n",
      "Creating a Migration Repository\n",
      "To begin, Flask-Migrate must be installed in the virtual environment:\n",
      "(venv) $ pip install flask-migrate\n",
      "Example 5-8 shows how the extension is initialized.\n",
      "Example 5-8. hello.py: Flask-Migrate initialization\n",
      "from flask_migrate import Migrate\n",
      "# ...\n",
      "migrate = Migrate(app, db)\n",
      "To expose the database migration commands, Flask-Migrate adds a flask db com‐\n",
      "mand with several subcommands. When you work on a new project, you can add\n",
      "support for database migrations with the init subcommand:\n",
      "(venv) $ flask db init\n",
      "  Creating directory /home/flask/flasky/migrations...done\n",
      "  Creating directory /home/flask/flasky/migrations/versions...done\n",
      "  Generating /home/flask/flasky/migrations/alembic.ini...done\n",
      "  Generating /home/flask/flasky/migrations/env.py...done\n",
      "  Generating /home/flask/flasky/migrations/env.pyc...done\n",
      "  Generating /home/flask/flasky/migrations/README...done\n",
      "  Generating /home/flask/flasky/migrations/script.py.mako...done\n",
      "  Please edit configuration/connection/logging settings in\n",
      "  '/home/flask/flasky/migrations/alembic.ini' before proceeding.\n",
      "Database Migrations with Flask-Migrate \n",
      "| \n",
      "73\n",
      "\n",
      "This command creates a migrations directory, where all the migration scripts will be\n",
      "stored. If you are following the example project using git checkout, you do not need\n",
      "to do this step, as the migration repository is already included in the GitHub reposi‐\n",
      "tory.\n",
      "The files in a database migration repository must always be added\n",
      "to version control along with the rest of the application.\n",
      "Creating a Migration Script\n",
      "In Alembic, a database migration is represented by a migration script. This script has\n",
      "two functions called upgrade() and downgrade(). The upgrade() function applies\n",
      "the database changes that are part of the migration, and the downgrade() function\n",
      "removes them. This ability to add and remove changes means, Alembic can reconfig‐\n",
      "ure a database to any point in the change history.\n",
      "Alembic migrations can be created manually or automatically using the revision and\n",
      "migrate commands, respectively. A manual migration creates a migration skeleton\n",
      "script with empty upgrade() and downgrade() functions that need to be imple‐\n",
      "mented by the developer using directives exposed by Alembic’s Operations object.\n",
      "An automatic migration attempts to generate the code for the upgrade() and\n",
      "downgrade() functions by looking for differences between the model definitions and\n",
      "the current state of the database.\n",
      "Automatic migrations are not always accurate and can miss some\n",
      "details that are ambiguous. For example, if a column is renamed, an\n",
      "automatically generated migration may show that the column in\n",
      "question was deleted and a new column was added with the new\n",
      "name. Leaving the migration as is will cause the data in this column\n",
      "to be lost! For this reason, migration scripts generated automati‐\n",
      "cally should always be reviewed and manually corrected if they\n",
      "have any inaccuracies.\n",
      "To make changes to your database schema with Flask-Migrate, the following proce‐\n",
      "dure needs to be followed:\n",
      "1. Make the necessary changes to the model classes.\n",
      "2. Create an automatic migration script with the flask db migrate command.\n",
      "3. Review the generated script and adjust it so that it accurately represents the\n",
      "changes that were made to the models.\n",
      "74 \n",
      "| \n",
      "Chapter 5: Databases\n",
      "\n",
      "4. Add the migration script to source control.\n",
      "5. Apply the migration to the database with the flask db upgrade command.\n",
      "The flask db migrate subcommand creates an automatic migration script:\n",
      "(venv) $ flask db migrate -m \"initial migration\"\n",
      "INFO  [alembic.migration] Context impl SQLiteImpl.\n",
      "INFO  [alembic.migration] Will assume non-transactional DDL.\n",
      "INFO  [alembic.autogenerate] Detected added table 'roles'\n",
      "INFO  [alembic.autogenerate] Detected added table 'users'\n",
      "INFO  [alembic.autogenerate.compare] Detected added index\n",
      "'ix_users_username' on '['username']'\n",
      "  Generating /home/flask/flasky/migrations/versions/1bc\n",
      "  594146bb5_initial_migration.py...done\n",
      "If you are following the git checkout instructions to incrementally update the exam‐\n",
      "ple application, you do not need to issue the migrate commands, as the migration\n",
      "scripts are already incorporated into the Git repository tags.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 5d to check out this version of the applica‐\n",
      "tion. Note that you do not need to generate the migration reposi‐\n",
      "tory and the migration scripts for this application as these are\n",
      "included in the GitHub repository.\n",
      "Upgrading the Database\n",
      "Once a migration script has been reviewed and accepted, it can be applied to the data‐\n",
      "base using the flask db upgrade command:\n",
      "(venv) $ flask db upgrade\n",
      "INFO  [alembic.migration] Context impl SQLiteImpl.\n",
      "INFO  [alembic.migration] Will assume non-transactional DDL.\n",
      "INFO  [alembic.migration] Running upgrade None -> 1bc594146bb5, initial migration\n",
      "For a first migration, this is effectively equivalent to calling db.create_all(), but in\n",
      "successive migrations the flask db upgrade command applies updates to the tables\n",
      "without affecting their contents.\n",
      "Database Migrations with Flask-Migrate \n",
      "| \n",
      "75\n",
      "\n",
      "If you have been working with the application in its previous\n",
      "stages, you already have a database file that was created with the\n",
      "db.create_all() function earlier. In this state, the flask db\n",
      "upgrade will fail because it will try to create database tables that\n",
      "already exist. A simple way to address this problem is to delete\n",
      "your data.sqlite database file and then run flask db upgrade to\n",
      "generate a new database through the migration framework.\n",
      "Another option is to skip the flask db upgrade and instead mark\n",
      "the existing database as upgraded using the flask db stamp com‐\n",
      "mand.\n",
      "Adding More Migrations\n",
      "As you work on your own projects, you are going to find that you need to make\n",
      "changes to your database models very often. When you manage the database through\n",
      "a migration framework, all changes must be defined in migration scripts, because\n",
      "anything that is not tracked in a migration will not be repeatable. The procedure to\n",
      "introduce a change in the database is similar to what was done to introduce the first\n",
      "migration:\n",
      "1. Make the necessary changes in the database models.\n",
      "2. Generate a migration with the flask db migrate command.\n",
      "3. Review the generated migration script and correct it if it has any inaccuracies.\n",
      "4. Apply the changes to the database with the flask db upgrade command.\n",
      "While working on a specific feature, you may find that you need to make several\n",
      "changes to your database models before you get them the way you want them. If your\n",
      "last migration has not been committed to source control yet, you can opt to expand it\n",
      "to incorporate new changes as you make them, and this will save you from having\n",
      "lots of very small migration scripts that are meaningless on their own. The procedure\n",
      "to expand the last migration script is as follows:\n",
      "1. Remove the last migration from the database with the flask db downgrade com‐\n",
      "mand (note that this may cause some data to be lost).\n",
      "2. Delete the last migration script, which is now orphaned.\n",
      "3. Generate a new database migration with the flask db migrate command,\n",
      "which will now include the changes in the migration script you just removed,\n",
      "plus any other changes you’ve made to the models.\n",
      "4. Review and apply the migration script as described previously.\n",
      "76 \n",
      "| \n",
      "Chapter 5: Databases\n",
      "\n",
      "Consult the Flask-Migrate documentation to learn about other\n",
      "subcommands related to database migrations.\n",
      "The topic of database design and usage is very important; entire books have been\n",
      "written on the subject. You should consider this chapter as an overview; more\n",
      "advanced topics will be discussed in later chapters. The next chapter is dedicated to\n",
      "sending email.\n",
      "Database Migrations with Flask-Migrate \n",
      "| \n",
      "77\n",
      "\n",
      "\n",
      "CHAPTER 6\n",
      "Email\n",
      "Many types of applications need to notify users when certain events occur, and the\n",
      "usual method of communication is email. In this chapter you are going to learn how\n",
      "to send emails from a Flask application.\n",
      "Email Support with Flask-Mail\n",
      "Although the smtplib package from the Python standard library can be used to send\n",
      "email inside a Flask application, the Flask-Mail extension wraps smtplib and integra‐\n",
      "tes it nicely with Flask. Flask-Mail is installed with pip:\n",
      "(venv) $ pip install flask-mail\n",
      "The extension connects to a Simple Mail Transfer Protocol (SMTP) server and passes\n",
      "emails to it for delivery. If no configuration is given, Flask-Mail connects to localhost\n",
      "at port 25 and sends email without authentication. Table 6-1 shows the list of configu‐\n",
      "ration keys that can be used to configure the SMTP server.\n",
      "Table 6-1. Flask-Mail SMTP server configuration keys\n",
      "Key\n",
      "Default\n",
      "Description\n",
      "MAIL_SERVER\n",
      "localhost\n",
      "Hostname or IP address of the email server\n",
      "MAIL_PORT\n",
      "25\n",
      "Port of the email server\n",
      "MAIL_USE_TLS\n",
      "False\n",
      "Enable Transport Layer Security (TLS) security\n",
      "MAIL_USE_SSL\n",
      "False\n",
      "Enable Secure Sockets Layer (SSL) security\n",
      "MAIL_USERNAME\n",
      "None\n",
      "Mail account username\n",
      "MAIL_PASSWORD\n",
      "None\n",
      "Mail account password\n",
      "79\n",
      "\n",
      "During development it may be more convenient to connect to an external SMTP\n",
      "server. As an example, Example 6-1 shows how to configure the application to send\n",
      "email through a Google Gmail account.\n",
      "Example 6-1. hello.py: Flask-Mail configuration for Gmail\n",
      "import os\n",
      "# ...\n",
      "app.config['MAIL_SERVER'] = 'smtp.googlemail.com'\n",
      "app.config['MAIL_PORT'] = 587\n",
      "app.config['MAIL_USE_TLS'] = True\n",
      "app.config['MAIL_USERNAME'] = os.environ.get('MAIL_USERNAME')\n",
      "app.config['MAIL_PASSWORD'] = os.environ.get('MAIL_PASSWORD')\n",
      "Never write account credentials directly in your scripts, particularly\n",
      "if you plan to release your work as open source. To protect your\n",
      "account information, have your script import sensitive information\n",
      "from environment variables.\n",
      "For security reasons, Gmail accounts are configured to require\n",
      "external applications to use OAuth2 authentication to connect to\n",
      "the email server. Unfortunately, Python’s smtplib library does not\n",
      "support this method of authentication. To make your Gmail\n",
      "account accept standard SMTP authentication, go to your Google\n",
      "account settings page and select “Signing in to Google” from the\n",
      "left menu bar. On that page, locate the “Allow less secure apps” set‐\n",
      "ting and make sure it is enabled. If enabling this setting on your\n",
      "personal Gmail account concerns you, create a secondary account\n",
      "only to test sending emails.\n",
      "Flask-Mail is initialized as shown in Example 6-2.\n",
      "Example 6-2. hello.py: Flask-Mail initialization\n",
      "from flask_mail import Mail\n",
      "mail = Mail(app)\n",
      "The two environment variables that hold the email server username and password\n",
      "need to be defined in the environment. If you are on Linux or macOS, you can set\n",
      "these variables as follows:\n",
      "(venv) $ export MAIL_USERNAME=<Gmail username>\n",
      "(venv) $ export MAIL_PASSWORD=<Gmail password>\n",
      "80 \n",
      "| \n",
      "Chapter 6: Email\n",
      "\n",
      "For Microsoft Windows users, the environment variables are set as follows:\n",
      "(venv) $ set MAIL_USERNAME=<Gmail username>\n",
      "(venv) $ set MAIL_PASSWORD=<Gmail password>\n",
      "Sending Email from the Python Shell\n",
      "To test the configuration, you can start a shell session and send a test email (replace\n",
      "you@example.com with your own email address):\n",
      "(venv) $ flask shell\n",
      ">>> from flask_mail import Message\n",
      ">>> from hello import mail\n",
      ">>> msg = Message('test email', sender='you@example.com',\n",
      "...     recipients=['you@example.com'])\n",
      ">>> msg.body = 'This is the plain text body'\n",
      ">>> msg.html = 'This is the <b>HTML</b> body'\n",
      ">>> with app.app_context():\n",
      "...    mail.send(msg)\n",
      "...\n",
      "Note that Flask-Mail’s send() function uses current_app, so it needs to be executed\n",
      "with an activated application context.\n",
      "Integrating Emails with the Application\n",
      "To avoid having to create email messages manually every time, it is a good idea to\n",
      "abstract the common parts of the application’s email sending functionality into a\n",
      "function. As an added benefit, this function can render email bodies from Jinja2 tem‐\n",
      "plates to have the most flexibility. The implementation is shown in Example 6-3.\n",
      "Example 6-3. hello.py: email support\n",
      "from flask_mail import Message\n",
      "app.config['FLASKY_MAIL_SUBJECT_PREFIX'] = '[Flasky]'\n",
      "app.config['FLASKY_MAIL_SENDER'] = 'Flasky Admin <flasky@example.com>'\n",
      "def send_email(to, subject, template, **kwargs):\n",
      "    msg = Message(app.config['FLASKY_MAIL_SUBJECT_PREFIX'] + subject,\n",
      "                  sender=app.config['FLASKY_MAIL_SENDER'], recipients=[to])\n",
      "    msg.body = render_template(template + '.txt', **kwargs)\n",
      "    msg.html = render_template(template + '.html', **kwargs)\n",
      "    mail.send(msg)\n",
      "The function relies on two application-specific configuration keys that define a prefix\n",
      "string for the subject and the address that will be used as the sender. The\n",
      "send_email() function takes the destination address, a subject line, a template for the\n",
      "email body, and a list of keyword arguments. The template name must be given\n",
      "Email Support with Flask-Mail \n",
      "| \n",
      "81\n",
      "\n",
      "without the extension, so that two versions of the template can be used for the plain\n",
      "text and HTML bodies. The keyword arguments passed by the caller are given to\n",
      "render_template() so that they can be used by the templates that generate the email\n",
      "body as template variables.\n",
      "The index() view function can easily be expanded to send an email to the adminis‐\n",
      "trator whenever a new name is received with the form. Example 6-4 shows this\n",
      "change.\n",
      "Example 6-4. hello.py: email example\n",
      "# ...\n",
      "app.config['FLASKY_ADMIN'] = os.environ.get('FLASKY_ADMIN')\n",
      "# ...\n",
      "@app.route('/', methods=['GET', 'POST'])\n",
      "def index():\n",
      "    form = NameForm()\n",
      "    if form.validate_on_submit():\n",
      "        user = User.query.filter_by(username=form.name.data).first()\n",
      "        if user is None:\n",
      "            user = User(username=form.name.data)\n",
      "            db.session.add(user)\n",
      "            session['known'] = False\n",
      "            if app.config['FLASKY_ADMIN']:\n",
      "                send_email(app.config['FLASKY_ADMIN'], 'New User',\n",
      "                           'mail/new_user', user=user)\n",
      "        else:\n",
      "            session['known'] = True\n",
      "        session['name'] = form.name.data\n",
      "        form.name.data = ''\n",
      "        return redirect(url_for('index'))\n",
      "    return render_template('index.html', form=form, name=session.get('name'),\n",
      "                           known=session.get('known', False))\n",
      "The recipient of the email is given in the FLASKY_ADMIN environment variable that’s\n",
      "loaded into a configuration variable of the same name during startup. Two template\n",
      "files need to be created for the text and HTML versions of the email. These files are\n",
      "stored in a mail subdirectory inside templates to keep them separate from regular\n",
      "templates. The email templates expect the user to be given as a template argument, so\n",
      "the call to send_email() includes it as a keyword argument.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 6a to check out this version of the applica‐\n",
      "tion.\n",
      "82 \n",
      "| \n",
      "Chapter 6: Email\n",
      "\n",
      "In addition to the MAIL_USERNAME and MAIL_PASSWORD environment variables\n",
      "described earlier, this version of the application needs the FLASKY_ADMIN environment\n",
      "variable. For Linux and macOS users, the command to set this variable is:\n",
      "(venv) $ export FLASKY_ADMIN=<your-email-address>\n",
      "For Microsoft Windows users, this is the equivalent command:\n",
      "(venv) $ set FLASKY_ADMIN=<your-email-address>\n",
      "With these environment variables set, you can test the application and receive an\n",
      "email every time you enter a new name in the form.\n",
      "Sending Asynchronous Email\n",
      "If you sent a few test emails, you likely noticed that the mail.send() function blocks\n",
      "for a few seconds while the email is sent, making the browser look unresponsive dur‐\n",
      "ing that time. To avoid unnecessary delays during request handling, the email send\n",
      "function can be moved to a background thread. Example 6-5 shows this change.\n",
      "Example 6-5. hello.py: asynchronous email support\n",
      "from threading import Thread\n",
      "def send_async_email(app, msg):\n",
      "    with app.app_context():\n",
      "        mail.send(msg)\n",
      "def send_email(to, subject, template, **kwargs):\n",
      "    msg = Message(app.config['FLASKY_MAIL_SUBJECT_PREFIX'] + subject,\n",
      "                  sender=app.config['FLASKY_MAIL_SENDER'], recipients=[to])\n",
      "    msg.body = render_template(template + '.txt', **kwargs)\n",
      "    msg.html = render_template(template + '.html', **kwargs)\n",
      "    thr = Thread(target=send_async_email, args=[app, msg])\n",
      "    thr.start()\n",
      "    return thr\n",
      "This implementation highlights an interesting problem. Many Flask extensions oper‐\n",
      "ate under the assumption that there are active application and/or request contexts. As\n",
      "mentioned previously, Flask-Mail’s send() function uses current_app, so it requires\n",
      "the application context to be active. But since contexts are associated with a thread,\n",
      "when the mail.send() function executes in a different thread it needs the application\n",
      "context to be created artificially using app.app_context(). The app instance is passed\n",
      "to the thread as an argument so that a context can be created.\n",
      "Email Support with Flask-Mail \n",
      "| \n",
      "83\n",
      "\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 6b to check out this version of the applica‐\n",
      "tion.\n",
      "If you run the application now, you will notice that it is much more responsive, but\n",
      "keep in mind that for applications that send a large volume of email, having a job\n",
      "dedicated to sending email is more appropriate than starting a new thread for each\n",
      "email send operation. For example, the execution of the send_async_email() func‐\n",
      "tion can be sent to a Celery task queue.\n",
      "This chapter completes the overview of the features that are a must-have for most\n",
      "web applications. The problem now is that the hello.py script is starting to get large,\n",
      "and that makes it harder to work with. In the next chapter, you will learn how to\n",
      "structure a larger application.\n",
      "84 \n",
      "| \n",
      "Chapter 6: Email\n",
      "\n",
      "CHAPTER 7\n",
      "Large Application Structure\n",
      "Although having small web applications stored in a single script file can be very con‐\n",
      "venient, this approach does not scale well. As the application grows in complexity,\n",
      "working with a single large source file becomes problematic.\n",
      "Unlike most other web frameworks, Flask does not impose a specific organization for\n",
      "large projects; the way to structure the application is left entirely to the developer. In\n",
      "this chapter, a possible way to organize a large application in packages and modules is\n",
      "presented. This structure will be used in the remaining examples of the book.\n",
      "Project Structure\n",
      "Example 7-1 shows the basic layout for a Flask application.\n",
      "Example 7-1. Basic multiple-file Flask application structure\n",
      "|-flasky\n",
      "  |-app/\n",
      "    |-templates/\n",
      "    |-static/\n",
      "    |-main/\n",
      "      |-__init__.py\n",
      "      |-errors.py\n",
      "      |-forms.py\n",
      "      |-views.py\n",
      "    |-__init__.py\n",
      "    |-email.py\n",
      "    |-models.py\n",
      "  |-migrations/\n",
      "  |-tests/\n",
      "    |-__init__.py\n",
      "    |-test*.py\n",
      "  |-venv/\n",
      "  |-requirements.txt\n",
      "  |-config.py\n",
      "  |-flasky.py\n",
      "85\n",
      "\n",
      "This structure has four top-level folders:\n",
      "• The Flask application lives inside a package generically named app.\n",
      "• The migrations folder contains the database migration scripts, as before.\n",
      "• Unit tests are written in a tests package.\n",
      "• The venv folder contains the Python virtual environment, as before.\n",
      "There are also a few new files:\n",
      "• requirements.txt lists the package dependencies so that it is easy to regenerate an\n",
      "identical virtual environment on a different computer.\n",
      "• config.py stores the configuration settings.\n",
      "• flasky.py defines the Flask application instance, and also includes a few tasks that\n",
      "help manage the application.\n",
      "To help you fully understand this structure, the following sections describe the pro‐\n",
      "cess to convert the hello.py application to it.\n",
      "Configuration Options\n",
      "Applications often need several configuration sets. The best example of this is the\n",
      "need to use different databases during development, testing, and production so that\n",
      "they don’t interfere with each other.\n",
      "Instead of the simple app.config dictionary-like configuration used by hello.py, a\n",
      "hierarchy of configuration classes can be used. Example 7-2 shows the config.py file,\n",
      "with all the settings imported from hello.py.\n",
      "Example 7-2. config.py: application configuration\n",
      "import os\n",
      "basedir = os.path.abspath(os.path.dirname(__file__))\n",
      "class Config:\n",
      "    SECRET_KEY = os.environ.get('SECRET_KEY') or 'hard to guess string'\n",
      "    MAIL_SERVER = os.environ.get('MAIL_SERVER', 'smtp.googlemail.com')\n",
      "    MAIL_PORT = int(os.environ.get('MAIL_PORT', '587'))\n",
      "    MAIL_USE_TLS = os.environ.get('MAIL_USE_TLS', 'true').lower() in \\\n",
      "        ['true', 'on', '1']\n",
      "    MAIL_USERNAME = os.environ.get('MAIL_USERNAME')\n",
      "    MAIL_PASSWORD = os.environ.get('MAIL_PASSWORD')\n",
      "    FLASKY_MAIL_SUBJECT_PREFIX = '[Flasky]'\n",
      "    FLASKY_MAIL_SENDER = 'Flasky Admin <flasky@example.com>'\n",
      "    FLASKY_ADMIN = os.environ.get('FLASKY_ADMIN')\n",
      "    SQLALCHEMY_TRACK_MODIFICATIONS = False\n",
      "86 \n",
      "| \n",
      "Chapter 7: Large Application Structure\n",
      "\n",
      "    @staticmethod\n",
      "    def init_app(app):\n",
      "        pass\n",
      "class DevelopmentConfig(Config):\n",
      "    DEBUG = True\n",
      "    SQLALCHEMY_DATABASE_URI = os.environ.get('DEV_DATABASE_URL') or \\\n",
      "        'sqlite:///' + os.path.join(basedir, 'data-dev.sqlite')\n",
      "class TestingConfig(Config):\n",
      "    TESTING = True\n",
      "    SQLALCHEMY_DATABASE_URI = os.environ.get('TEST_DATABASE_URL') or \\\n",
      "        'sqlite://'\n",
      "class ProductionConfig(Config):\n",
      "    SQLALCHEMY_DATABASE_URI = os.environ.get('DATABASE_URL') or \\\n",
      "        'sqlite:///' + os.path.join(basedir, 'data.sqlite')\n",
      "config = {\n",
      "    'development': DevelopmentConfig,\n",
      "    'testing': TestingConfig,\n",
      "    'production': ProductionConfig,\n",
      "    'default': DevelopmentConfig\n",
      "}\n",
      "The Config base class contains settings that are common to all configurations; the\n",
      "different subclasses define settings that are specific to a configuration. Additional\n",
      "configurations can be added as needed.\n",
      "To make configuration more flexible and safe, most settings can be optionally impor‐\n",
      "ted from environment variables. For example, the value of the SECRET_KEY, due to its\n",
      "sensitive nature, can be set in the environment, but a default value is provided in case\n",
      "the environment does not define it. Typically, these settings can be used with their\n",
      "defaults during development but should each have an appropriate value set in the\n",
      "corresponding environment variable on the production server. The configuration\n",
      "options for the email server are all imported from environment variables as well, with\n",
      "defaults pointing to the Gmail server for convenience during development.\n",
      "Never write passwords or other secrets in a configuration file that is\n",
      "committed to source control.\n",
      "The SQLALCHEMY_DATABASE_URI variable is assigned different values under each of the\n",
      "three configurations. This enables the application to use a different database in each\n",
      "Configuration Options \n",
      "| \n",
      "87\n",
      "\n",
      "configuration. This is very important, as you don’t want a run of the unit tests to\n",
      "change the database that you use for day-to-day development. Each configuration\n",
      "tries to import the database URL from an environment variable, and when that is not\n",
      "available it sets a default one based on SQLite. For the testing configuration, the\n",
      "default is an in-memory database, since there is no need to store any data outside of\n",
      "the test run.\n",
      "The development and production configurations each have a set of mail server con‐\n",
      "figuration options. As an additional way to allow the application to customize its con‐\n",
      "figuration, the Config class and its subclasses can define an init_app() class method\n",
      "that takes the application instance as an argument. For now the base Config class\n",
      "implements an empty init_app() method.\n",
      "At the bottom of the configuration script, the different configurations are registered\n",
      "in a config dictionary. One of the configurations (the one for development, in this\n",
      "case) is also registered as the default.\n",
      "Application Package\n",
      "The application package is where all the application code, templates, and static files\n",
      "live. It is called simply app here, though it can be given an application-specific name if\n",
      "desired. The templates and static directories are now part of the application package,\n",
      "so they are moved inside app. The database models and the email support functions\n",
      "are also moved inside this package, each in its own module, as app/models.py and\n",
      "app/email.py.\n",
      "Using an Application Factory\n",
      "The way the application is created in the single-file version is very convenient, but it\n",
      "has one big drawback. Because the application is created in the global scope, there is\n",
      "no way to apply configuration changes dynamically: by the time the script is running,\n",
      "the application instance has already been created, so it is already too late to make\n",
      "configuration changes. This is particularly important for unit tests because sometimes\n",
      "it is necessary to run the application under different configuration settings for better\n",
      "test coverage.\n",
      "The solution to this problem is to delay the creation of the application by moving it\n",
      "into a factory function that can be explicitly invoked from the script. This not only\n",
      "gives the script time to set the configuration, but also the ability to create multiple\n",
      "application instances—another thing that can be very useful during testing. The\n",
      "application factory function, shown in Example 7-3, is defined in the app package\n",
      "constructor.\n",
      "88 \n",
      "| \n",
      "Chapter 7: Large Application Structure\n",
      "\n",
      "Example 7-3. app/__init__.py: application package constructor\n",
      "from flask import Flask, render_template\n",
      "from flask_bootstrap import Bootstrap\n",
      "from flask_mail import Mail\n",
      "from flask_moment import Moment\n",
      "from flask_sqlalchemy import SQLAlchemy\n",
      "from config import config\n",
      "bootstrap = Bootstrap()\n",
      "mail = Mail()\n",
      "moment = Moment()\n",
      "db = SQLAlchemy()\n",
      "def create_app(config_name):\n",
      "    app = Flask(__name__)\n",
      "    app.config.from_object(config[config_name])\n",
      "    config[config_name].init_app(app)\n",
      "    bootstrap.init_app(app)\n",
      "    mail.init_app(app)\n",
      "    moment.init_app(app)\n",
      "    db.init_app(app)\n",
      "    # attach routes and custom error pages here\n",
      "    return app\n",
      "This constructor imports most of the Flask extensions currently in use, but because\n",
      "there is no application instance to initialize them with, it creates them uninitialized\n",
      "by passing no arguments into their constructors. The create_app() function is the\n",
      "application factory, which takes as an argument the name of a configuration to use\n",
      "for the application. The configuration settings stored in one of the classes defined in\n",
      "config.py can be imported directly into the application using the from_object()\n",
      "method available in Flask’s app.config configuration object. The configuration\n",
      "object is selected by name from the config dictionary. Once an application is created\n",
      "and configured, the extensions can be initialized. Calling init_app() on the exten‐\n",
      "sions that were created earlier completes their initialization.\n",
      "The application initialization is now done in this factory function, using the\n",
      "from_object() method from the Flask configuration object, which takes as an argu‐\n",
      "ment one of the configuration classes defined in config.py. The init_app() method of\n",
      "the selected configuration is also invoked, to allow more complex initialization proce‐\n",
      "dures to take place.\n",
      "The factory function returns the created application instance, but note that applica‐\n",
      "tions created with the factory function in its current state are incomplete, as they are\n",
      "missing routes and custom error page handlers. This is the topic of the next section.\n",
      "Application Package \n",
      "| \n",
      "89\n",
      "\n",
      "Implementing Application Functionality in a Blueprint\n",
      "The conversion to an application factory introduces a complication for routes. In\n",
      "single-script applications, the application instance exists in the global scope, so routes\n",
      "can be easily defined using the app.route decorator. But now that the application is\n",
      "created at runtime, the app.route decorator begins to exist only after create_app()\n",
      "is invoked, which is too late. Custom error page handlers present the same problem,\n",
      "as these are defined with the app.errorhandler decorator.\n",
      "Luckily, Flask offers a better solution using blueprints. A blueprint is similar to an\n",
      "application in that it can also define routes and error handlers. The difference is that\n",
      "when these are defined in a blueprint they are in a dormant state until the blueprint is\n",
      "registered with an application, at which point they become part of it. Using a blue‐\n",
      "print defined in the global scope, the routes and error handlers of the application can\n",
      "be defined in almost the same way as in the single-script application.\n",
      "Like applications, blueprints can be defined all in a single file or can be created in a\n",
      "more structured way with multiple modules inside a package. To allow for the great‐\n",
      "est flexibility, a subpackage inside the application package will be created to host the\n",
      "first blueprint of the application. Example 7-4 shows the package constructor, which\n",
      "creates the blueprint.\n",
      "Example 7-4. app/main/__init__.py: main blueprint creation\n",
      "from flask import Blueprint\n",
      "main = Blueprint('main', __name__)\n",
      "from . import views, errors\n",
      "Blueprints are created by instantiating an object of class Blueprint. The constructor\n",
      "for this class takes two required arguments: the blueprint name and the module or\n",
      "package where the blueprint is located. As with applications, Python’s __name__ vari‐\n",
      "able is in most cases the correct value for the second argument.\n",
      "The routes of the application are stored in an app/main/views.py module inside the\n",
      "package, and the error handlers are in app/main/errors.py. Importing these modules\n",
      "causes the routes and error handlers to be associated with the blueprint. It is impor‐\n",
      "tant to note that the modules are imported at the bottom of the app/main/__init__.py\n",
      "script to avoid errors due to circular dependencies. In this particular example the\n",
      "problem is that app/main/views.py and app/main/errors.py in turn are going to import\n",
      "the main blueprint object, so the imports are going to fail unless the circular reference\n",
      "occurs after main is defined.\n",
      "90 \n",
      "| \n",
      "Chapter 7: Large Application Structure\n",
      "\n",
      "The from . import <some-module> syntax is used in Python to\n",
      "represent relative imports. The . in this statement represents the\n",
      "current package. You are going to see another very useful relative\n",
      "import soon that uses the form from .. import <some-module>,\n",
      "where .. represents the parent of the current package.\n",
      "The blueprint is registered with the application inside the create_app() factory func‐\n",
      "tion, as shown in Example 7-5.\n",
      "Example 7-5. app/__init__.py: main blueprint registration\n",
      "def create_app(config_name):\n",
      "    # ...\n",
      "    from .main import main as main_blueprint\n",
      "    app.register_blueprint(main_blueprint)\n",
      "    return app\n",
      "Example 7-6 shows the error handlers.\n",
      "Example 7-6. app/main/errors.py: error handlers in main blueprint\n",
      "from flask import render_template\n",
      "from . import main\n",
      "@main.app_errorhandler(404)\n",
      "def page_not_found(e):\n",
      "    return render_template('404.html'), 404\n",
      "@main.app_errorhandler(500)\n",
      "def internal_server_error(e):\n",
      "    return render_template('500.html'), 500\n",
      "A difference when writing error handlers inside a blueprint is that if the\n",
      "errorhandler decorator is used, the handler will be invoked only for errors that orig‐\n",
      "inate in the routes defined by the blueprint. To install application-wide error han‐\n",
      "dlers, the app_errorhandler decorator must be used instead.\n",
      "Example 7-7 shows the route of the application updated to be in the blueprint.\n",
      "Application Package \n",
      "| \n",
      "91\n",
      "\n",
      "Example 7-7. app/main/views.py: application routes in main blueprint\n",
      "from datetime import datetime\n",
      "from flask import render_template, session, redirect, url_for\n",
      "from . import main\n",
      "from .forms import NameForm\n",
      "from .. import db\n",
      "from ..models import User\n",
      "@main.route('/', methods=['GET', 'POST'])\n",
      "def index():\n",
      "    form = NameForm()\n",
      "    if form.validate_on_submit():\n",
      "        # ...\n",
      "        return redirect(url_for('.index'))\n",
      "    return render_template('index.html',\n",
      "                           form=form, name=session.get('name'),\n",
      "                           known=session.get('known', False),\n",
      "                           current_time=datetime.utcnow())\n",
      "There are two main differences when writing a view function inside a blueprint. First,\n",
      "as was done for error handlers earlier, the route decorator comes from the blueprint,\n",
      "so main.route is used instead of app.route. The second difference is in the usage of\n",
      "the url_for() function. As you may recall, the first argument to this function is the\n",
      "endpoint name of the route, which for application-based routes defaults to the name\n",
      "of the view function. For example, in a single-script application the URL for an\n",
      "index() view function can be obtained with url_for('index').\n",
      "The difference with blueprints is that Flask applies a namespace to all the endpoints\n",
      "defined in a blueprint, so that multiple blueprints can define view functions with the\n",
      "same endpoint names without collisions. The namespace is the name of the blueprint\n",
      "(the first argument to the Blueprint constructor) and is separated from the endpoint\n",
      "name with a dot. The index() view function is then registered with endpoint name\n",
      "main.index and its URL can be obtained with url_for('main.index').\n",
      "The url_for() function also supports a shorter format for endpoints in blueprints in\n",
      "which the blueprint name is omitted, such as url_for('.index'). With this nota‐\n",
      "tion, the blueprint name for the current request is used to complete the endpoint\n",
      "name. This effectively means that redirects within the same blueprint can use the\n",
      "shorter form, while redirects across blueprints must use the fully qualified endpoint\n",
      "name that includes the blueprint name.\n",
      "To complete the changes to the application package, the form objects are also stored\n",
      "inside the blueprint in the app/main/forms.py module.\n",
      "92 \n",
      "| \n",
      "Chapter 7: Large Application Structure\n",
      "\n",
      "Application Script\n",
      "The flasky.py module in the top-level directory is where the application instance is\n",
      "defined. This script is shown in Example 7-8.\n",
      "Example 7-8. flasky.py: main script\n",
      "import os\n",
      "from app import create_app, db\n",
      "from app.models import User, Role\n",
      "from flask_migrate import Migrate\n",
      "app = create_app(os.getenv('FLASK_CONFIG') or 'default')\n",
      "migrate = Migrate(app, db)\n",
      "@app.shell_context_processor\n",
      "def make_shell_context():\n",
      "    return dict(db=db, User=User, Role=Role)\n",
      "The script begins by creating an application. The configuration is taken from the\n",
      "environment variable FLASK_CONFIG if it’s defined, or else the default configuration is\n",
      "used. Flask-Migrate and the custom context for the Python shell are then initialized.\n",
      "Because the main script of the application changed from hello.py to flasky.py, the\n",
      "FLASK_APP environment variable needs to be updated accordingly so that the flask\n",
      "command can locate the application instance. It is also useful to enable Flask’s debug\n",
      "mode by setting FLASK_DEBUG=1. For Linux and macOS, this is all done as follows:\n",
      "(venv) $ export FLASK_APP=flasky.py\n",
      "(venv) $ export FLASK_DEBUG=1\n",
      "And for Microsoft Windows:\n",
      "(venv) $ set FLASK_APP=flasky.py\n",
      "(venv) $ set FLASK_DEBUG=1\n",
      "Requirements File\n",
      "It is a good practice for applications to include a requirements.txt file that records all\n",
      "the package dependencies, with the exact version numbers. This is important in case\n",
      "the virtual environment needs to be regenerated on a different machine, such as the\n",
      "machine on which the application will be deployed for production use. This file can\n",
      "be generated automatically by pip with the following command:\n",
      "(venv) $ pip freeze >requirements.txt\n",
      "It is a good idea to refresh this file whenever a package is installed or upgraded. An\n",
      "example requirements file is shown here:\n",
      "Application Script \n",
      "| \n",
      "93\n",
      "\n",
      "alembic==0.9.3\n",
      "blinker==1.4\n",
      "click==6.7\n",
      "dominate==2.3.1\n",
      "Flask==0.12.2\n",
      "Flask-Bootstrap==3.3.7.1\n",
      "Flask-Mail==0.9.1\n",
      "Flask-Migrate==2.0.4\n",
      "Flask-Moment==0.5.1\n",
      "Flask-SQLAlchemy==2.2\n",
      "Flask-WTF==0.14.2\n",
      "itsdangerous==0.24\n",
      "Jinja2==2.9.6\n",
      "Mako==1.0.7\n",
      "MarkupSafe==1.0\n",
      "python-dateutil==2.6.1\n",
      "python-editor==1.0.3\n",
      "six==1.10.0\n",
      "SQLAlchemy==1.1.11\n",
      "visitor==0.1.3\n",
      "Werkzeug==0.12.2\n",
      "WTForms==2.1\n",
      "When you need to build a perfect replica of the virtual environment, you can create a\n",
      "new virtual environment and run the following command on it:\n",
      "(venv) $ pip install -r requirements.txt\n",
      "The version numbers in the example requirements.txt file are likely going to be outda‐\n",
      "ted by the time you read this. You can try using more recent releases of the packages,\n",
      "if you like. If you experience any problems, you can always go back to the versions\n",
      "specified here, as those are known to be compatible with the application.\n",
      "Unit Tests\n",
      "This application is very small, so there isn’t a lot to test yet. But as an example, two\n",
      "simple tests can be defined, as shown in Example 7-9.\n",
      "Example 7-9. tests/test_basics.py: unit tests\n",
      "import unittest\n",
      "from flask import current_app\n",
      "from app import create_app, db\n",
      "class BasicsTestCase(unittest.TestCase):\n",
      "    def setUp(self):\n",
      "        self.app = create_app('testing')\n",
      "        self.app_context = self.app.app_context()\n",
      "        self.app_context.push()\n",
      "        db.create_all()\n",
      "94 \n",
      "| \n",
      "Chapter 7: Large Application Structure\n",
      "\n",
      "    def tearDown(self):\n",
      "        db.session.remove()\n",
      "        db.drop_all()\n",
      "        self.app_context.pop()\n",
      "    def test_app_exists(self):\n",
      "        self.assertFalse(current_app is None)\n",
      "    def test_app_is_testing(self):\n",
      "        self.assertTrue(current_app.config['TESTING'])\n",
      "The tests are written using the standard unittest package from the Python standard\n",
      "library. The setUp() and tearDown() methods of the test case class run before and\n",
      "after each test, and any methods that have a name that begins with test_ are executed\n",
      "as tests.\n",
      "If you want to learn more about writing unit tests with Python’s\n",
      "unittest package, read the official documentation.\n",
      "The setUp() method tries to create an environment for the test that is close to that of\n",
      "a running application. It first creates an application configured for testing and acti‐\n",
      "vates its context. This step ensures that tests have access to current_app, like regular\n",
      "requests do. Then it creates a brand-new database for the tests using Flask-\n",
      "SQLAlchemy’s create_all() method. The database and the application context are\n",
      "removed in the tearDown() method.\n",
      "The first test ensures that the application instance exists. The second test ensures that\n",
      "the application is running under the testing configuration. To make the tests directory\n",
      "a proper package, a tests/init.py module needs to be added, but this can be an empty\n",
      "file, as the unittest package scans all the modules to discover the tests.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 7a to check out the converted version of the\n",
      "application. To ensure that you have all the dependencies installed,\n",
      "also run pip install -r requirements.txt.\n",
      "To run the unit tests, a custom command can be added to the flasky.py script.\n",
      "Example 7-10 shows how to add a test command.\n",
      "Unit Tests \n",
      "| \n",
      "95\n",
      "\n",
      "Example 7-10. flasky.py: unit test launcher command\n",
      "@app.cli.command()\n",
      "def test():\n",
      "    \"\"\"Run the unit tests.\"\"\"\n",
      "    import unittest\n",
      "    tests = unittest.TestLoader().discover('tests')\n",
      "    unittest.TextTestRunner(verbosity=2).run(tests)\n",
      "The app.cli.command decorator makes it simple to implement custom commands.\n",
      "The name of the decorated function is used as the command name, and the function’s\n",
      "docstring is displayed in the help messages. The implementation of the test() func‐\n",
      "tion invokes the test runner from the unittest package.\n",
      "The unit tests can be executed as follows:\n",
      "(venv) $ flask test\n",
      "test_app_exists (test_basics.BasicsTestCase) ... ok\n",
      "test_app_is_testing (test_basics.BasicsTestCase) ... ok\n",
      ".----------------------------------------------------------------------\n",
      "Ran 2 tests in 0.001s\n",
      "OK\n",
      "----\n",
      "Database Setup\n",
      "The restructured application uses a different database than the single-script version.\n",
      "The database URL is taken from an environment variable as a first choice, with a\n",
      "default SQLite database as an alternative. The environment variables and SQLite\n",
      "database filenames are different for each of the three configurations. For example, in\n",
      "the development configuration the URL is obtained from the environment variable\n",
      "DEV_DATABASE_URL, and if that is not defined then an SQLite database with the name\n",
      "data-dev.sqlite is used.\n",
      "Regardless of the source of the database URL, the database tables must be created for\n",
      "the new database. When working with Flask-Migrate to keep track of migrations,\n",
      "database tables can be created or upgraded to the latest revision with a single com‐\n",
      "mand:\n",
      "(venv) $ flask db upgrade\n",
      "96 \n",
      "| \n",
      "Chapter 7: Large Application Structure\n",
      "\n",
      "Running the Application\n",
      "The refactoring is now complete, and the application can be started. Make sure you\n",
      "have updated the FLASK_APP environment variable as indicated in “Application Script”\n",
      "on page 93, and then run the application as usual:\n",
      "(venv) $ flask run\n",
      "Having to set the FLASK_APP and FLASK_DEBUG environment variables every time a\n",
      "new command-prompt session is started can get tedious, so you should configure\n",
      "your system so that these variables are set by default. If you are using bash, you can\n",
      "add them to your ~/.bashrc file.\n",
      "Believe it or not, you have reached the end of Part I. You have now learned about the\n",
      "basic elements necessary to build a web application with Flask, but you probably feel\n",
      "unsure about how all these pieces fit together to form a real application. The goal of\n",
      "Part II is to help with that by walking you through the development of a complete\n",
      "application.\n",
      "Running the Application \n",
      "| \n",
      "97\n",
      "\n",
      "\n",
      "PART II\n",
      "Example: A Social\n",
      "Blogging Application\n",
      "\n",
      "\n",
      "CHAPTER 8\n",
      "User Authentication\n",
      "Most applications need to keep track of who their users are. When users connect with\n",
      "an application, they authenticate with it, a process by which they make their identity\n",
      "known. Once the application knows who the user is, it can offer a customized experi‐\n",
      "ence.\n",
      "The most commonly used method of authentication requires users to provide a piece\n",
      "of identification, which is either their email address or username, and a secret only\n",
      "known to them, which is called the password. In this chapter, the complete authenti‐\n",
      "cation system for Flasky is created.\n",
      "Authentication Extensions for Flask\n",
      "There are many excellent Python authentication packages, but none of them do\n",
      "everything. The user authentication solution presented in this chapter uses several\n",
      "packages and provides the glue that makes them work well together. This is the list of\n",
      "packages that will be used, and what they’re used for:\n",
      "• Flask-Login: Management of user sessions for logged-in users\n",
      "• Werkzeug: Password hashing and verification\n",
      "• itsdangerous: Cryptographically secure token generation and verification\n",
      "In addition to authentication-specific packages, the following general-purpose exten‐\n",
      "sions will be used:\n",
      "• Flask-Mail: Sending of authentication-related emails\n",
      "• Flask-Bootstrap: HTML templates\n",
      "• Flask-WTF: Web forms\n",
      "101\n",
      "\n",
      "Password Security\n",
      "The safety of user information stored in databases is often overlooked during the\n",
      "design of web applications. If an attacker is able to break into your server and access\n",
      "your user database, then you risk the security of your users—and the risk is bigger\n",
      "than you think. It is a known fact that most users use the same password on multiple\n",
      "sites, so even if you don’t store any sensitive information, access to the passwords\n",
      "stored in your database can give the attacker access to accounts your users have on\n",
      "other sites.\n",
      "The key to storing user passwords securely in a database relies on not storing the\n",
      "password itself but a hash of it. A password hashing function takes a password as\n",
      "input, adds a random component to it (the salt), and then applies several one-way\n",
      "cryptographic transformations to it. The result is a new sequence of characters that\n",
      "has no resemblance to the original password, and has no known way to be trans‐\n",
      "formed back into the original password. Password hashes can be verified in place of\n",
      "the real passwords because hashing functions are repeatable: given the same inputs\n",
      "(the password and the salt), the result is always the same.\n",
      "Password hashing is a complex task that is hard to get right. It is\n",
      "recommended that you don’t implement your own solution but\n",
      "instead rely on well-known libraries that have been reviewed by the\n",
      "community. In the next section, Werkzeug’s password hashing\n",
      "functions will be demonstrated. Other good choices for password\n",
      "hashing are bcrypt and Passlib. If you are interested in learning\n",
      "what’s involved in generating secure password hashes, the article\n",
      "“Salted Password Hashing - Doing It Right” by Defuse Security is a\n",
      "worthwhile read.\n",
      "Hashing Passwords with Werkzeug\n",
      "Werkzeug’s security module conveniently implements secure password hashing. This\n",
      "functionality is exposed with just two functions, used in the registration and verifica‐\n",
      "tion phases, respectively:\n",
      "generate_password_hash(password, method='pbkdf2:sha256', salt_length=8)\n",
      "This function takes a plain-text password and returns the password hash as a\n",
      "string that can be stored in the user database. The default values for method and\n",
      "salt_length are sufficient for most use cases.\n",
      "check_password_hash(hash, password)\n",
      "This function takes a password hash previously stored in the database and the\n",
      "password entered by the user. A return value of True indicates that the user pass‐\n",
      "word is correct.\n",
      "102 \n",
      "| \n",
      "Chapter 8: User Authentication\n",
      "\n",
      "Example 8-1 shows the changes to the User model created in Chapter 5 to accommo‐\n",
      "date password hashing.\n",
      "Example 8-1. app/models.py: password hashing in the User model\n",
      "from werkzeug.security import generate_password_hash, check_password_hash\n",
      "class User(db.Model):\n",
      "    # ...\n",
      "    password_hash = db.Column(db.String(128))\n",
      "    @property\n",
      "    def password(self):\n",
      "        raise AttributeError('password is not a readable attribute')\n",
      "    @password.setter\n",
      "    def password(self, password):\n",
      "        self.password_hash = generate_password_hash(password)\n",
      "    def verify_password(self, password):\n",
      "        return check_password_hash(self.password_hash, password)\n",
      "The password hashing function is implemented through a write-only property called\n",
      "password. When this property is set, the setter method will call Werkzeug’s\n",
      "generate_password_hash() function and write the result to the password_hash field.\n",
      "Attempting to read the password property will return an error, as clearly the original\n",
      "password cannot be recovered once hashed.\n",
      "The verify_password() method takes a password and passes it to Werkzeug’s\n",
      "check_password_hash() function for verification against the hashed version stored\n",
      "in the User model. If this method returns True, then the password is correct.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 8a to check out this version of the applica‐\n",
      "tion.\n",
      "Password Security \n",
      "| \n",
      "103\n",
      "\n",
      "The password hashing functionality is now complete and can be tested in the shell:\n",
      "(venv) $ flask shell\n",
      ">>> u = User()\n",
      ">>> u.password = 'cat'\n",
      ">>> u.password\n",
      "Traceback (most recent call last):\n",
      "  File \"<console>\", line 1, in <module>\n",
      "  File \"/home/flask/flasky/app/models.py\", line 24, in password\n",
      "    raise AttributeError('password is not a readable attribute')\n",
      "AttributeError: password is not a readable attribute\n",
      ">>> u.password_hash\n",
      "'pbkdf2:sha256:50000$moHwFH1B$ef1574909f9c549285e8547cad181c5e0213cfa44a4aba4349\n",
      "fa830aa1fd227f'\n",
      ">>> u.verify_password('cat')\n",
      "True\n",
      ">>> u.verify_password('dog')\n",
      "False\n",
      ">>> u2 = User()\n",
      ">>> u2.password = 'cat'\n",
      ">>> u2.password_hash\n",
      "'pbkdf2:sha256:50000$Pfz0m0KU$27be930b7f0e0119d38e8d8a62f7f5e75c0a7db61ae16709bc\n",
      "aa6cfd60c44b74'\n",
      "Note how trying to access the password property of a user returns an\n",
      "AttributeError. Also, users u and u2 have completely different password hashes,\n",
      "even though they both use the same password. To ensure that this functionality con‐\n",
      "tinues to work in the future, the preceding tests done manually can be written as unit\n",
      "tests that can be repeated easily. In Example 8-2 a new module inside the tests package\n",
      "is shown with three new tests that exercise the recent changes to the User model.\n",
      "Example 8-2. tests/test_user_model.py: password hashing tests\n",
      "import unittest\n",
      "from app.models import User\n",
      "class UserModelTestCase(unittest.TestCase):\n",
      "    def test_password_setter(self):\n",
      "        u = User(password = 'cat')\n",
      "        self.assertTrue(u.password_hash is not None)\n",
      "    def test_no_password_getter(self):\n",
      "        u = User(password = 'cat')\n",
      "        with self.assertRaises(AttributeError):\n",
      "            u.password\n",
      "    def test_password_verification(self):\n",
      "        u = User(password = 'cat')\n",
      "        self.assertTrue(u.verify_password('cat'))\n",
      "        self.assertFalse(u.verify_password('dog'))\n",
      "104 \n",
      "| \n",
      "Chapter 8: User Authentication\n",
      "\n",
      "    def test_password_salts_are_random(self):\n",
      "        u = User(password='cat')\n",
      "        u2 = User(password='cat')\n",
      "        self.assertTrue(u.password_hash != u2.password_hash)\n",
      "To run these new unit tests, use the following command:\n",
      "(venv) $ flask test\n",
      "test_app_exists (test_basics.BasicsTestCase) ... ok\n",
      "test_app_is_testing (test_basics.BasicsTestCase) ... ok\n",
      "test_no_password_getter (test_user_model.UserModelTestCase) ... ok\n",
      "test_password_salts_are_random (test_user_model.UserModelTestCase) ... ok\n",
      "test_password_setter (test_user_model.UserModelTestCase) ... ok\n",
      "test_password_verification (test_user_model.UserModelTestCase) ... ok\n",
      ".----------------------------------------------------------------------\n",
      "Ran 6 tests in 0.379s\n",
      "OK\n",
      "You can run the unit test suite like this every time you want to confirm everything is\n",
      "working as expected. Having the automation in place makes verifying this feature\n",
      "very low cost, so testing should be repeated often, to ensure that this functionality\n",
      "does not break in the future.\n",
      "Creating an Authentication Blueprint\n",
      "Blueprints were introduced in Chapter 7 as a way to define routes in the global scope\n",
      "after the creation of the application was moved into a factory function. In this sec‐\n",
      "tion, the routes related to the user authentication subsystem will be added to a second\n",
      "blueprint, called auth. Using different blueprints for different subsystems of the\n",
      "application is a great way to keep the code neatly organized.\n",
      "The auth blueprint will be hosted in a Python package with the same name. The blue‐\n",
      "print’s package constructor creates the blueprint object and imports routes from a\n",
      "views.py module. This is shown in Example 8-3.\n",
      "Example 8-3. app/auth/__init__.py: authentication blueprint creation\n",
      "from flask import Blueprint\n",
      "auth = Blueprint('auth', __name__)\n",
      "from . import views\n",
      "The app/auth/views.py module, shown in Example 8-4, imports the blueprint and\n",
      "defines the routes associated with authentication using its route decorator. For now,\n",
      "a /login route is added, which renders a placeholder template of the same name.\n",
      "Creating an Authentication Blueprint \n",
      "| \n",
      "105\n",
      "\n",
      "Example 8-4. app/auth/views.py: authentication blueprint routes and view functions\n",
      "from flask import render_template\n",
      "from . import auth\n",
      "@auth.route('/login')\n",
      "def login():\n",
      "    return render_template('auth/login.html')\n",
      "Note that the template file given to render_template() is stored inside the auth\n",
      "directory. This directory must be created inside app/templates, as Flask expects the\n",
      "templates’ paths to be relative to the application’s templates directory. By storing the\n",
      "blueprint templates in their own subdirectory, there is no risk of naming collisions\n",
      "with the main blueprint or any other blueprints that will be added in the future.\n",
      "Blueprints can also be configured to have their own independent\n",
      "directories for templates. When multiple template directories have\n",
      "been configured, the render_template() function searches the\n",
      "templates directory configured for the application first, and then\n",
      "searches the template directories defined by blueprints.\n",
      "The auth blueprint needs to be attached to the application in the create_app() fac‐\n",
      "tory function, as shown in Example 8-5.\n",
      "Example 8-5. app/__init__.py: authentication blueprint registration\n",
      "def create_app(config_name):\n",
      "    # ...\n",
      "    from .auth import auth as auth_blueprint\n",
      "    app.register_blueprint(auth_blueprint, url_prefix='/auth')\n",
      "    return app\n",
      "The url_prefix argument in the blueprint registration is optional. When used, all\n",
      "the routes defined in the blueprint will be registered with the given prefix, in this\n",
      "case /auth. For example, the /login route will be registered as /auth/login, and the fully\n",
      "qualified URL under the development web server then becomes http://localhost:5000/\n",
      "auth/login.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 8b to check out this version of the applica‐\n",
      "tion.\n",
      "106 \n",
      "| \n",
      "Chapter 8: User Authentication\n",
      "\n",
      "User Authentication with Flask-Login\n",
      "When users log in to the application, their authenticated state has to be recorded in\n",
      "the user session, so that it is remembered as they navigate through different pages.\n",
      "Flask-Login is a small but extremely useful extension that specializes in managing\n",
      "this particular aspect of a user authentication system, without being tied to a specific\n",
      "authentication mechanism.\n",
      "To begin, the extension needs to be installed in the virtual environment:\n",
      "(venv) $ pip install flask-login\n",
      "Preparing the User Model for Logins\n",
      "Flask-Login works closely with the application’s own User objects. To be able to work\n",
      "with the application’s User model, the Flask-Login extension requires it to implement\n",
      "a few common properties and methods. The required items are shown in Table 8-1.\n",
      "Table 8-1. Flask-Login required items\n",
      "Property/method\n",
      "Description\n",
      "is_authenticated Must be True if the user has valid login credentials or False otherwise.\n",
      "is_active\n",
      "Must be True if the user is allowed to log in or False otherwise. A False value can be used for\n",
      "disabled accounts.\n",
      "is_anonymous\n",
      "Must always be False for regular users and True for a special user object that represents\n",
      "anonymous users.\n",
      "get_id()\n",
      "Must return a unique identifier for the user, encoded as a Unicode string.\n",
      "These properties and methods can be implemented directly in the model class, but as\n",
      "an easier alternative Flask-Login provides a UserMixin class that has default imple‐\n",
      "mentations that are appropriate for most cases. The updated User model is shown in\n",
      "Example 8-6.\n",
      "Example 8-6. app/models.py: updates to the User model to support user logins\n",
      "from flask_login import UserMixin\n",
      "class User(UserMixin, db.Model):\n",
      "    __tablename__ = 'users'\n",
      "    id = db.Column(db.Integer, primary_key = True)\n",
      "    email = db.Column(db.String(64), unique=True, index=True)\n",
      "    username = db.Column(db.String(64), unique=True, index=True)\n",
      "    password_hash = db.Column(db.String(128))\n",
      "    role_id = db.Column(db.Integer, db.ForeignKey('roles.id'))\n",
      "User Authentication with Flask-Login \n",
      "| \n",
      "107\n",
      "\n",
      "Note that an email field was also added. In this application, users will log in with\n",
      "their email addresses, as they are less likely to forget those than their usernames.\n",
      "Flask-Login is initialized in the application factory function, as shown in\n",
      "Example 8-7.\n",
      "Example 8-7. app/__init__.py: Flask-Login initialization\n",
      "from flask_login import LoginManager\n",
      "login_manager = LoginManager()\n",
      "login_manager.login_view = 'auth.login'\n",
      "def create_app(config_name):\n",
      "    # ...\n",
      "    login_manager.init_app(app)\n",
      "    # ...\n",
      "The login_view attribute of the LoginManager object sets the endpoint for the login\n",
      "page. Flask-Login will redirect to the login page when an anonymous user tries to\n",
      "access a protected page. Because the login route is inside a blueprint, it needs to be\n",
      "prefixed with the blueprint name.\n",
      "Finally, Flask-Login requires the application to designate a function to be invoked\n",
      "when the extension needs to load a user from the database given its identifier. This\n",
      "function is shown in Example 8-8.\n",
      "Example 8-8. app/models.py: user loader function\n",
      "from . import login_manager\n",
      "@login_manager.user_loader\n",
      "def load_user(user_id):\n",
      "    return User.query.get(int(user_id))\n",
      "The login_manager.user_loader decorator is used to register the function with\n",
      "Flask-Login, which will call it when it needs to retrieve information about the logged-\n",
      "in user. The user identifier will be passed as a string, so the function converts it to an\n",
      "integer before it passes it to the Flask-SQLAlchemy query that loads the user. The\n",
      "return value of the function must be the user object, or None if the user identifier is\n",
      "invalid or any other error occurred.\n",
      "Protecting Routes\n",
      "To protect a route so that it can only be accessed by authenticated users, Flask-Login\n",
      "provides a login_required decorator. An example of its usage follows:\n",
      "108 \n",
      "| \n",
      "Chapter 8: User Authentication\n",
      "\n",
      "from flask_login import login_required\n",
      "@app.route('/secret')\n",
      "@login_required\n",
      "def secret():\n",
      "    return 'Only authenticated users are allowed!'\n",
      "You can see from this example that it is possible to “chain” multiple function decora‐\n",
      "tors. When two or more decorators are added to a function, each decorator only\n",
      "affects those that are below it, in addition to the target function. In this example, the\n",
      "secret() \n",
      "function \n",
      "will \n",
      "be \n",
      "protected \n",
      "against \n",
      "unauthorized \n",
      "users \n",
      "with\n",
      "login_required, and then the resulting function will be registered with Flask as a\n",
      "route. Reversing the order will produce the wrong result, as the original function will\n",
      "be registered as a route before it receives the additional properties from the\n",
      "login_required decorator.\n",
      "Thanks to the login_required decorator, if this route is accessed by a user who is not\n",
      "authenticated, Flask-Login will intercept the request and send the user to the login\n",
      "page instead.\n",
      "Adding a Login Form\n",
      "The login form that will be presented to users has a text field for the email address, a\n",
      "password field, a “remember me” checkbox, and a submit button. The Flask-WTF\n",
      "form class that defines this form is shown in Example 8-9.\n",
      "Example 8-9. app/auth/forms.py: login form\n",
      "from flask_wtf import FlaskForm\n",
      "from wtforms import StringField, PasswordField, BooleanField, SubmitField\n",
      "from wtforms.validators import DataRequired, Length, Email\n",
      "class LoginForm(FlaskForm):\n",
      "    email = StringField('Email', validators=[DataRequired(), Length(1, 64),\n",
      "                                             Email()])\n",
      "    password = PasswordField('Password', validators=[DataRequired()])\n",
      "    remember_me = BooleanField('Keep me logged in')\n",
      "    submit = SubmitField('Log In')\n",
      "The PasswordField class represents an <input> element with type=\"password\". The\n",
      "BooleanField class represents a checkbox.\n",
      "The email field uses the Length() and Email() validators from WTForms in addition\n",
      "to DataRequired(), to ensure that the user not only provides a value for this field, but\n",
      "that it is valid. When providing a list of validators, WTForms will evaluate them in\n",
      "the order provided, and in case of a validation failure the error message shown will be\n",
      "the one of the first validator that failed.\n",
      "User Authentication with Flask-Login \n",
      "| \n",
      "109\n",
      "\n",
      "The template associated with the login page is stored in auth/login.html. This tem‐\n",
      "plate just needs to render the form using Flask-Bootstrap’s wtf.quick_form() macro.\n",
      "Figure 8-1 shows the login form rendered by the web browser.\n",
      "Figure 8-1. Login form\n",
      "The navigation bar in the base.html template uses a Jinja2 conditional to display “Log\n",
      "In” or “Log Out” links depending on the logged-in state of the current user. The con‐\n",
      "ditional is shown in Example 8-10.\n",
      "Example 8-10. app/templates/base.html: Log In and Log Out navigation bar links\n",
      "<ul class=\"nav navbar-nav navbar-right\">\n",
      "    {% if current_user.is_authenticated %}\n",
      "    <li><a href=\"{{ url_for('auth.logout') }}\">Log Out</a></li>\n",
      "    {% else %}\n",
      "    <li><a href=\"{{ url_for('auth.login') }}\">Log In</a></li>\n",
      "    {% endif %}\n",
      "</ul>\n",
      "The current_user variable used in the conditional is defined by Flask-Login and is\n",
      "automatically available to view functions and templates. This variable contains the\n",
      "currently logged-in user, or a proxy anonymous user object if the user is not logged\n",
      "in. Anonymous user objects have the is_authenticated property set to False, so the\n",
      "110 \n",
      "| \n",
      "Chapter 8: User Authentication\n",
      "\n",
      "expression current_user.is_authenticated is a convenient way to know whether\n",
      "the current user is logged in.\n",
      "Signing Users In\n",
      "The implementation of the login() view function is shown in Example 8-11.\n",
      "Example 8-11. app/auth/views.py: login route\n",
      "from flask import render_template, redirect, request, url_for, flash\n",
      "from flask_login import login_user\n",
      "from . import auth\n",
      "from ..models import User\n",
      "from .forms import LoginForm\n",
      "@auth.route('/login', methods=['GET', 'POST'])\n",
      "def login():\n",
      "    form = LoginForm()\n",
      "    if form.validate_on_submit():\n",
      "        user = User.query.filter_by(email=form.email.data).first()\n",
      "        if user is not None and user.verify_password(form.password.data):\n",
      "            login_user(user, form.remember_me.data)\n",
      "            next = request.args.get('next')\n",
      "            if next is None or not next.startswith('/'):\n",
      "                next = url_for('main.index')\n",
      "            return redirect(next)\n",
      "        flash('Invalid username or password.')\n",
      "    return render_template('auth/login.html', form=form)\n",
      "The view function creates a LoginForm object and uses it like the simple form in\n",
      "Chapter 4. When the request is of type GET, the view function just renders the tem‐\n",
      "plate, which in turn displays the form. When the form is submitted in a POST request,\n",
      "Flask-WTF’s validate_on_submit() function validates the form variables, and then\n",
      "attempts to log the user in.\n",
      "To log a user in, the function begins by loading the user from the database using the\n",
      "email provided with the form. If a user with the given email address exists, then its\n",
      "verify_password() method is called with the password that also came with the form.\n",
      "If the password is valid, Flask-Login’s login_user() function is invoked to record the\n",
      "user as logged in for the user session. The login_user() function takes the user to\n",
      "log in and an optional “remember me” Boolean, which was also submitted with the\n",
      "form. A value of False for this argument causes the user session to expire when the\n",
      "browser window is closed, so the user will have to log in again next time. A value of\n",
      "True causes a long-term cookie to be set in the user’s browser, which Flask-Login uses\n",
      "to restore the user session. The optional REMEMBER_COOKIE_DURATION configuration\n",
      "option can be used to change the default one-year duration for the remember cookie.\n",
      "User Authentication with Flask-Login \n",
      "| \n",
      "111\n",
      "\n",
      "In accordance with the Post/Redirect/Get pattern discussed in Chapter 4, the POST\n",
      "request that submitted the login credentials ends with a redirect, but there are two\n",
      "possible URL destinations. If the login form was presented to the user to prevent\n",
      "unauthorized access to a protected URL the user wanted to visit, then Flask-Login\n",
      "will have saved that original URL in the next query string argument, which can be\n",
      "accessed from the request.args dictionary. If the next query string argument is not\n",
      "available, a redirect to the home page is issued instead. The URL in next is validated\n",
      "to make sure it is a relative URL, to prevent a malicious user from using this argu‐\n",
      "ment to redirect unsuspecting users to another site.\n",
      "For the case where the email address or password provided by the user is invalid, a\n",
      "flash message is set and the form is rendered again for the user to retry.\n",
      "On a production server, the application must be made available\n",
      "over secure HTTP, so that login credentials and user sessions are\n",
      "always transmitted encrypted. Without secure HTTP, sensitive data\n",
      "can be intercepted during transit by an attacker.\n",
      "The login template needs to be updated to render the form. These changes are shown\n",
      "in Example 8-12.\n",
      "Example 8-12. app/templates/auth/login.html: login form template\n",
      "{% extends \"base.html\" %}\n",
      "{% import \"bootstrap/wtf.html\" as wtf %}\n",
      "{% block title %}Flasky - Login{% endblock %}\n",
      "{% block page_content %}\n",
      "<div class=\"page-header\">\n",
      "    <h1>Login</h1>\n",
      "</div>\n",
      "<div class=\"col-md-4\">\n",
      "    {{ wtf.quick_form(form) }}\n",
      "</div>\n",
      "{% endblock %}\n",
      "Signing Users Out\n",
      "The implementation of the logout route is shown in Example 8-13.\n",
      "Example 8-13. app/auth/views.py: logout route\n",
      "from flask_login import logout_user, login_required\n",
      "@auth.route('/logout')\n",
      "112 \n",
      "| \n",
      "Chapter 8: User Authentication\n",
      "\n",
      "@login_required\n",
      "def logout():\n",
      "    logout_user()\n",
      "    flash('You have been logged out.')\n",
      "    return redirect(url_for('main.index'))\n",
      "To log a user out, Flask-Login’s logout_user() function is called to remove and reset\n",
      "the user session. The logout is completed with a flash message that confirms the\n",
      "action and a redirect to the home page.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 8c to check out this version of the applica‐\n",
      "tion. This update contains a database migration, so remember to\n",
      "run flask db upgrade after you check out the code. To ensure that\n",
      "you have all the dependencies installed, also run pip install -r\n",
      "requirements.txt.\n",
      "Understanding How Flask-Login Works\n",
      "Flask-Login is a fairly small extension, but due to the many moving pieces involved in\n",
      "the authentication flow, Flask users often have trouble understanding how the exten‐\n",
      "sion works. The following is the sequence of operations that occur when a user logs in\n",
      "to the system:\n",
      "1. The user navigates to http://localhost:5000/auth/login by clicking on the “Log In”\n",
      "link. The handler for this URL returns the login form template.\n",
      "2. The user enters their username and password, and presses the Submit button.\n",
      "The same handler is invoked again, but now as a POST request instead of GET.\n",
      "a. The handler validates the credentials submitted with the form, and then\n",
      "invokes Flask-Login’s login_user() function to log the user in.\n",
      "b. The login_user() function writes the ID of the user to the user session as a\n",
      "string.\n",
      "c. The view function returns with a redirect to the home page.\n",
      "3. The browser receives the redirect and requests the home page.\n",
      "a. The view function for the home page is invoked, and it triggers the rendering\n",
      "of the main Jinja2 template.\n",
      "b. During the rendering of the Jinja2 template, a reference to Flask-Login’s\n",
      "current_user appears for the first time.\n",
      "c. The current_user context variable does not have a value assigned for this\n",
      "request yet, so it invokes Flask-Login’s internal function _get_user() to find\n",
      "out who the user is.\n",
      "User Authentication with Flask-Login \n",
      "| \n",
      "113\n",
      "\n",
      "d. The _get_user() function checks if there is a user ID stored in the user ses‐\n",
      "sion. If there isn’t one, it returns an instance of Flask-Login’s AnonymousUser.\n",
      "If there is an ID, it invokes the function that the application registered with\n",
      "the user_loader decorator, with the ID as its argument.\n",
      "e. The application’s user_loader handler reads the user from the database and\n",
      "returns it. Flask-Login assigns it to the current_user context variable for the\n",
      "current request.\n",
      "f. The template receives the newly assigned value of current_user.\n",
      "The login_required decorator builds on top of the current_user context variable\n",
      "by only allowing the decorated view function to run when the expression\n",
      "current_user.is_authenticated is True. The logout_user() function simply dele‐\n",
      "tes the user ID from the user session.\n",
      "Testing Logins\n",
      "To verify that the login functionality is working, the home page can be updated to\n",
      "greet the logged-in user by name. The template section that generates the greeting is\n",
      "shown in Example 8-14.\n",
      "Example 8-14. app/templates/index.html: greeting the logged-in user\n",
      "Hello,\n",
      "{% if current_user.is_authenticated %}\n",
      "    {{ current_user.username }}\n",
      "{% else %}\n",
      "    Stranger\n",
      "{% endif %}!\n",
      "In this template once again current_user.is_authenticated is used to determine\n",
      "whether the user is logged in.\n",
      "Because no user registration functionality has been built, a new user can only be reg‐\n",
      "istered from the shell at this time:\n",
      "(venv) $ $ flask shell\n",
      ">>> u = User(email='john@example.com', username='john', password='cat')\n",
      ">>> db.session.add(u)\n",
      ">>> db.session.commit()\n",
      "The user created previously can now log in. Figure 8-2 shows the application home\n",
      "page with the user logged in.\n",
      "114 \n",
      "| \n",
      "Chapter 8: User Authentication\n",
      "\n",
      "Figure 8-2. Home page after successful login\n",
      "New User Registration\n",
      "When new users want to become members of the application, they must register with\n",
      "it so that they are known and can log in. A link in the login page will send them to a\n",
      "registration page, where they can enter their email address, username, and password.\n",
      "Adding a User Registration Form\n",
      "The form that will be used in the registration page asks the user to enter an email\n",
      "address, username, and password. This form is shown in Example 8-15.\n",
      "Example 8-15. app/auth/forms.py: user registration form\n",
      "from flask_wtf import FlaskForm\n",
      "from wtforms import StringField, PasswordField, BooleanField, SubmitField\n",
      "from wtforms.validators import DataRequired, Length, Email, Regexp, EqualTo\n",
      "from wtforms import ValidationError\n",
      "from ..models import User\n",
      "class RegistrationForm(FlaskForm):\n",
      "    email = StringField('Email', validators=[DataRequired(), Length(1, 64),\n",
      "                                             Email()])\n",
      "New User Registration \n",
      "| \n",
      "115\n",
      "\n",
      "    username = StringField('Username', validators=[\n",
      "        DataRequired(), Length(1, 64),\n",
      "        Regexp('^[A-Za-z][A-Za-z0-9_.]*$', 0,\n",
      "               'Usernames must have only letters, numbers, dots or '\n",
      "               'underscores')])\n",
      "    password = PasswordField('Password', validators=[\n",
      "        DataRequired(), EqualTo('password2', message='Passwords must match.')])\n",
      "    password2 = PasswordField('Confirm password', validators=[DataRequired()])\n",
      "    submit = SubmitField('Register')\n",
      "    def validate_email(self, field):\n",
      "        if User.query.filter_by(email=field.data).first():\n",
      "            raise ValidationError('Email already registered.')\n",
      "    def validate_username(self, field):\n",
      "        if User.query.filter_by(username=field.data).first():\n",
      "            raise ValidationError('Username already in use.')\n",
      "This form uses the Regexp validator from WTForms to ensure that the username field\n",
      "starts with a letter and only contains letters, numbers, underscores, and dots. The two\n",
      "arguments to the validator that follow the regular expression are the regular expres‐\n",
      "sion flags and the error message to display on failure.\n",
      "The password is entered twice as a safety measure, but this step makes it necessary to\n",
      "validate that the two password fields have the same content, which is done with\n",
      "another validator from WTForms called EqualTo. This validator is attached to one of\n",
      "the password fields with the name of the other field given as an argument.\n",
      "This form also has two custom validators implemented as methods. When a form\n",
      "defines a method with the prefix validate_ followed by the name of a field, the\n",
      "method is invoked in addition to any regularly defined validators. In this case, the\n",
      "custom validators for email and username ensure that the values given are not dupli‐\n",
      "cates. The custom validators indicate a validation error by raising a ValidationError\n",
      "exception with the text of the error message as an argument.\n",
      "The template that presents this form is called /templates/auth/register.html. Like the\n",
      "login template, this one also renders the form with wtf.quick_form(). The registra‐\n",
      "tion page is shown in Figure 8-3.\n",
      "116 \n",
      "| \n",
      "Chapter 8: User Authentication\n",
      "\n",
      "Figure 8-3. New user registration form\n",
      "The registration page needs to be linked from the login page so that users who don’t\n",
      "have an account can easily find it. This change is shown in Example 8-16.\n",
      "Example 8-16. app/templates/auth/login.html: link to the registration page\n",
      "<p>\n",
      "    New user?\n",
      "    <a href=\"{{ url_for('auth.register') }}\">\n",
      "        Click here to register\n",
      "    </a>\n",
      "</p>\n",
      "Registering New Users\n",
      "Handling user registrations does not present any big surprises. When the registration\n",
      "form is submitted and validated, a new user is added to the database using the infor‐\n",
      "mation provided by the user. The view function that performs this task is shown in\n",
      "Example 8-17.\n",
      "New User Registration \n",
      "| \n",
      "117\n",
      "\n",
      "Example 8-17. app/auth/views.py: user registration route\n",
      "@auth.route('/register', methods=['GET', 'POST'])\n",
      "def register():\n",
      "    form = RegistrationForm()\n",
      "    if form.validate_on_submit():\n",
      "        user = User(email=form.email.data,\n",
      "                    username=form.username.data,\n",
      "                    password=form.password.data)\n",
      "        db.session.add(user)\n",
      "        db.session.commit()\n",
      "        flash('You can now login.')\n",
      "        return redirect(url_for('auth.login'))\n",
      "    return render_template('auth/register.html', form=form)\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 8d to check out this version of the applica‐\n",
      "tion.\n",
      "Account Confirmation\n",
      "For certain types of applications, it is important to ensure that the user information\n",
      "provided during registration is valid. A common requirement is to ensure that the\n",
      "user can be reached through the provided email address.\n",
      "To validate the email address, applications send a confirmation email to users imme‐\n",
      "diately after they register. The new account is initially marked as unconfirmed until\n",
      "the instructions in the email are followed, which proves that the user has received the\n",
      "email. The account confirmation procedure usually involves clicking a specially craf‐\n",
      "ted URL link that includes a confirmation token.\n",
      "Generating Confirmation Tokens with itsdangerous\n",
      "The simplest account confirmation link would be a URL with the format http://\n",
      "www.example.com/auth/confirm/<id> included in the confirmation email, where\n",
      "<id> is the numeric id assigned to the user in the database. When the user clicks the\n",
      "link, the view function that handles this route receives the user id to confirm as an\n",
      "argument and can easily update the confirmed status of the user.\n",
      "But this is obviously not a secure implementation, as any user who figures out the\n",
      "format of the confirmation links will be able to confirm arbitrary accounts just by\n",
      "sending random numbers in the URL. The idea is to replace the <id> in the URL\n",
      "with a token that contains the same information, but in such a way that only the\n",
      "server can generate valid confirmation URLs.\n",
      "118 \n",
      "| \n",
      "Chapter 8: User Authentication\n",
      "\n",
      "If you recall the discussion on user sessions in Chapter 4, Flask uses cryptographically\n",
      "signed cookies to protect the content of user sessions against tampering. The user ses‐\n",
      "sion cookies contain a cryptographic signature generated by a package called\n",
      "itsdangerous. If the contents of the user session is altered, the signature will not\n",
      "match the content anymore, so Flask discards the session and starts a new one. The\n",
      "same concept can be applied to confirmation tokens.\n",
      "The following is a short shell session that shows how itsdangerous can generate a\n",
      "signed token that contains a user id inside:\n",
      "(venv) $ flask shell\n",
      ">>> from itsdangerous import TimedJSONWebSignatureSerializer as Serializer\n",
      ">>> s = Serializer(app.config['SECRET_KEY'], expires_in=3600)\n",
      ">>> token = s.dumps({ 'confirm': 23 })\n",
      ">>> token\n",
      "'eyJhbGciOiJIUzI1NiIsImV4cCI6MTM4MTcxODU1OCwiaWF0IjoxMzgxNzE0OTU4fQ.ey ...'\n",
      ">>> data = s.loads(token)\n",
      ">>> data\n",
      "{'confirm': 23}\n",
      "The itsdangerous package provides several types of token generators. Among them,\n",
      "the class TimedJSONWebSignatureSerializer generates JSON Web Signatures (JWSs)\n",
      "with a time expiration. The constructor of this class takes an encryption key as an\n",
      "argument, which in a Flask application can be the configured SECRET_KEY.\n",
      "The dumps() method generates a cryptographic signature for the data given as an\n",
      "argument and then serializes the data plus the signature as a convenient token string.\n",
      "The expires_in argument sets an expiration time for the token, expressed in sec‐\n",
      "onds.\n",
      "To decode the token, the serializer object provides a loads() method that takes the\n",
      "token as its only argument. The function verifies the signature and the expiration\n",
      "time and, if both are valid, it returns the original data. When the loads() method is\n",
      "given an invalid token or a valid token that is expired, an exception is raised.\n",
      "Token generation and verification using this functionality can be added to the User\n",
      "model. The changes are shown in Example 8-18.\n",
      "Example 8-18. app/models.py: user account confirmation\n",
      "from itsdangerous import TimedJSONWebSignatureSerializer as Serializer\n",
      "from flask import current_app\n",
      "from . import db\n",
      "class User(UserMixin, db.Model):\n",
      "    # ...\n",
      "    confirmed = db.Column(db.Boolean, default=False)\n",
      "Account Confirmation \n",
      "| \n",
      "119\n",
      "\n",
      "    def generate_confirmation_token(self, expiration=3600):\n",
      "        s = Serializer(current_app.config['SECRET_KEY'], expiration)\n",
      "        return s.dumps({'confirm': self.id}).decode('utf-8')\n",
      "    def confirm(self, token):\n",
      "        s = Serializer(current_app.config['SECRET_KEY'])\n",
      "        try:\n",
      "            data = s.loads(token.encode('utf-8'))\n",
      "        except:\n",
      "            return False\n",
      "        if data.get('confirm') != self.id:\n",
      "            return False\n",
      "        self.confirmed = True\n",
      "        db.session.add(self)\n",
      "        return True\n",
      "The generate_confirmation_token() method generates a token with a default val‐\n",
      "idity time of one hour. The confirm() method verifies the token and, if valid, sets the\n",
      "new confirmed attribute in the user model to True.\n",
      "In addition to verifying the token, the confirm() function checks that the id from\n",
      "the token matches the logged-in user, which is stored in current_user. This ensures\n",
      "that a confirmation token for a given user cannot be used to confirm a different user.\n",
      "Because a new column was added to the model to track the con‐\n",
      "firmed state of each account, a new database migration needs to be\n",
      "generated and applied.\n",
      "The two new methods added to the User model are easily tested in unit tests. You can\n",
      "find the unit tests in the GitHub repository for the application.\n",
      "Sending Confirmation Emails\n",
      "The current /register route redirects to /index after adding the new user to the data‐\n",
      "base. Before redirecting, this route now needs to send the confirmation email. This\n",
      "change is shown in Example 8-19.\n",
      "Example 8-19. app/auth/views.py: registration route with confirmation email\n",
      "from ..email import send_email\n",
      "@auth.route('/register', methods=['GET', 'POST'])\n",
      "def register():\n",
      "    form = RegistrationForm()\n",
      "    if form.validate_on_submit():\n",
      "        # ...\n",
      "120 \n",
      "| \n",
      "Chapter 8: User Authentication\n",
      "\n",
      "        db.session.add(user)\n",
      "        db.session.commit()\n",
      "        token = user.generate_confirmation_token()\n",
      "        send_email(user.email, 'Confirm Your Account',\n",
      "                   'auth/email/confirm', user=user, token=token)\n",
      "        flash('A confirmation email has been sent to you by email.')\n",
      "        return redirect(url_for('main.index'))\n",
      "    return render_template('auth/register.html', form=form)\n",
      "Note that a db.session.commit() call had to be added before the confirmation email\n",
      "is sent out. The problem is that new users get assigned an id when they are commit‐\n",
      "ted to the database, and this id is needed to generate the confirmation token.\n",
      "The email templates used by the authentication blueprint will be added in the tem‐\n",
      "plates/auth/email directory to keep them separate from the HTML templates. As dis‐\n",
      "cussed in Chapter 6, for each email two templates are needed for the plain-text and\n",
      "HTML versions of the body. As an example, Example 8-20 shows the plain-text ver‐\n",
      "sion of the confirmation email template, and you can find the equivalent HTML ver‐\n",
      "sion in the GitHub repository.\n",
      "Example 8-20. app/templates/auth/email/confirm.txt: text body of confirmation email\n",
      "Dear {{ user.username }},\n",
      "Welcome to Flasky!\n",
      "To confirm your account please click on the following link:\n",
      "{{ url_for('auth.confirm', token=token, _external=True) }}\n",
      "Sincerely,\n",
      "The Flasky Team\n",
      "Note: replies to this email address are not monitored.\n",
      "By \n",
      "default, \n",
      "url_for() \n",
      "generates \n",
      "relative \n",
      "URLs; \n",
      "so, \n",
      "for \n",
      "example,\n",
      "url_for('auth.confirm', token='abc') returns the string '/auth/confirm/abc'.\n",
      "This, of course, is not a valid URL that can be sent in an email, since it is only the\n",
      "path portion of the URL. Relative URLs work fine when they are used within the con‐\n",
      "text of a web page because the browser converts them to absolute URLs by adding the\n",
      "hostname and port number from the current page, but when sending a URL over\n",
      "email there is no such context. The _external=True argument is added to the\n",
      "url_for() call to request a fully qualified URL that includes the scheme (http:// or\n",
      "https://), hostname, and port.\n",
      "The view function that confirms accounts is shown in Example 8-21.\n",
      "Account Confirmation \n",
      "| \n",
      "121\n",
      "\n",
      "Example 8-21. app/auth/views.py: confirming a user account\n",
      "from flask_login import current_user\n",
      "@auth.route('/confirm/<token>')\n",
      "@login_required\n",
      "def confirm(token):\n",
      "    if current_user.confirmed:\n",
      "        return redirect(url_for('main.index'))\n",
      "    if current_user.confirm(token):\n",
      "        db.session.commit()\n",
      "        flash('You have confirmed your account. Thanks!')\n",
      "    else:\n",
      "        flash('The confirmation link is invalid or has expired.')\n",
      "    return redirect(url_for('main.index'))\n",
      "This route is protected with the login_required decorator from Flask-Login, so that\n",
      "when the users click on the link from the confirmation email they are asked to log in\n",
      "before they reach this view function.\n",
      "The function first checks if the logged-in user is already confirmed, and in that case it\n",
      "redirects to the home page, as obviously there is nothing to do. This can prevent\n",
      "unnecessary work if a user clicks the confirmation token multiple times by mistake.\n",
      "Because the actual token confirmation is done entirely in the User model, all the view\n",
      "function needs to do is call the confirm() method and then flash a message accord‐\n",
      "ing to the result. When the confirmation succeeds, the User model’s confirmed\n",
      "attribute is changed and added to the session and then the database session is com‐\n",
      "mitted.\n",
      "Each application can decide what unconfirmed users are allowed to do before they\n",
      "confirm their accounts. One possibility is to allow unconfirmed users to log in, but\n",
      "only show them a page that asks them to confirm their accounts before they can gain\n",
      "further access.\n",
      "This step can be done using Flask’s before_request hook, which was briefly\n",
      "described in Chapter 2. From a blueprint, the before_request hook applies only to\n",
      "requests that belong to the blueprint. To install a blueprint hook for all application\n",
      "requests, the before_app_request decorator must be used instead. Example 8-22\n",
      "shows how this handler is implemented.\n",
      "122 \n",
      "| \n",
      "Chapter 8: User Authentication\n",
      "\n",
      "Example 8-22. app/auth/views.py: filtering unconfirmed accounts with the\n",
      "before_app_request handler\n",
      "@auth.before_app_request\n",
      "def before_request():\n",
      "    if current_user.is_authenticated \\\n",
      "            and not current_user.confirmed \\\n",
      "            and request.blueprint != 'auth' \\\n",
      "            and request.endpoint != 'static':\n",
      "        return redirect(url_for('auth.unconfirmed'))\n",
      "@auth.route('/unconfirmed')\n",
      "def unconfirmed():\n",
      "    if current_user.is_anonymous or current_user.confirmed:\n",
      "        return redirect(url_for('main.index'))\n",
      "    return render_template('auth/unconfirmed.html')\n",
      "The before_app_request handler will intercept a request when three conditions are\n",
      "true:\n",
      "1. A user is logged in (current_user.is_authenticated is True).\n",
      "2. The account for the user is not confirmed.\n",
      "3. The requested URL is outside of the authentication blueprint and is not for a\n",
      "static file. Access to the authentication routes needs to be granted, as those are the\n",
      "routes that will enable the user to confirm the account or perform other account\n",
      "management functions.\n",
      "If these three conditions are met, then a redirect is issued to a new /auth/unconfirmed\n",
      "route that shows a page with information about account confirmation.\n",
      "When a before_request or before_app_request callback returns\n",
      "a response or a redirect, Flask sends that to the client without\n",
      "invoking the view function associated with the request. This effec‐\n",
      "tively allows these callbacks to intercept a request when necessary.\n",
      "The page that is presented to unconfirmed users (shown in Figure 8-4) just renders a\n",
      "template that gives users instructions for how to confirm their accounts and offers a\n",
      "link to request a new confirmation email, in case the original email was lost. The\n",
      "route that resends the confirmation email is shown in Example 8-23.\n",
      "Account Confirmation \n",
      "| \n",
      "123\n",
      "\n",
      "Figure 8-4. Unconfirmed account page\n",
      "Example 8-23. app/auth/views.py: resending the account confirmation email\n",
      "@auth.route('/confirm')\n",
      "@login_required\n",
      "def resend_confirmation():\n",
      "    token = current_user.generate_confirmation_token()\n",
      "    send_email(current_user.email, 'Confirm Your Account',\n",
      "               'auth/email/confirm', user=current_user, token=token)\n",
      "    flash('A new confirmation email has been sent to you by email.')\n",
      "    return redirect(url_for('main.index'))\n",
      "This route repeats what was done in the registration route using current_user, the\n",
      "user who is logged in, as the target user. This route is also protected with\n",
      "login_required to ensure that when it is accessed, the user that is making the\n",
      "request is authenticated.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 8e to check out this version of the applica‐\n",
      "tion. This update contains a database migration, so remember to\n",
      "run flask db upgrade after you check out the code.\n",
      "124 \n",
      "| \n",
      "Chapter 8: User Authentication\n",
      "\n",
      "Account Management\n",
      "Users who have accounts with the application may need to make changes to their\n",
      "accounts from time to time. The following tasks can be added to the authentication\n",
      "blueprint using the techniques presented in this chapter:\n",
      "Password updates\n",
      "Security-conscious users may want to change their passwords periodically. This is\n",
      "an easy feature to implement, because as long as the user is logged in, it is safe to\n",
      "present a form that asks for the old password and a new password to replace it.\n",
      "This feature is implemented as commit 8f in the GitHub repository. As part of\n",
      "this change, the “Log Out” link in the navigation bar was refactored into a drop‐\n",
      "down that contains the “Change Password” and “Log Out” links.\n",
      "Password resets\n",
      "To avoid locking users out of the application when they forget their passwords, a\n",
      "password reset option can be offered. To implement password resets in a secure\n",
      "way, it is necessary to use tokens similar to those used to confirm accounts.\n",
      "When a user requests a password reset, an email with a reset token is sent to the\n",
      "registered email address. The user then clicks the link in the email and, after the\n",
      "token is verified, a form is presented where a new password can be entered. This\n",
      "feature is implemented as commit 8g in the GitHub repository.\n",
      "Email address changes\n",
      "Users can be given the option to change their registered email address, but before\n",
      "the new address is accepted it must be verified with a confirmation email. To use\n",
      "this feature, the user enters the new email address in a form. To confirm the\n",
      "email address, a token is emailed to that address. When the server receives the\n",
      "token back, it can update the user object. While the server waits to receive the\n",
      "token, it can store the new email address in a new database field reserved for\n",
      "pending email addresses, or it can store the address in the token along with the\n",
      "id. This feature is implemented as commit 8h in the GitHub repository.\n",
      "In the next chapter, the user subsystem of Flasky will be extended through the use of\n",
      "user roles.\n",
      "Account Management \n",
      "| \n",
      "125\n",
      "\n",
      "\n",
      "CHAPTER 9\n",
      "User Roles\n",
      "Not all users of web applications are created equal. In most applications, a small per‐\n",
      "centage of users are trusted with extra powers to help keep the application running\n",
      "smoothly. Administrators are the best example, but in many cases middle-level power\n",
      "users such as content moderators exist as well. To implement this, all users are\n",
      "assigned a role.\n",
      "There are several ways to implement roles in an application. The appropriate method\n",
      "largely depends on how many roles need to be supported and how elaborate they are.\n",
      "For example, a simple application may need just two roles, one for regular users and\n",
      "one for administrators. In this case, having an is_administrator Boolean field in the\n",
      "User model may be all that is necessary. A more complex application may need addi‐\n",
      "tional roles with varying levels of power in between regular users and administrators.\n",
      "In some applications it may not even make sense to talk about discrete roles, and\n",
      "instead giving users a set of individual permissions may be the right approach.\n",
      "The user role implementation presented in this chapter is a hybrid between discrete\n",
      "roles and permissions. Users are assigned a discrete role, but each role defines what\n",
      "actions it allows its users to perform through a list of permissions.\n",
      "Database Representation of Roles\n",
      "A simple roles table was created in Chapter 5 as a vehicle to demonstrate one-to-\n",
      "many relationships. Example 9-1 shows an improved Role model with some addi‐\n",
      "tions.\n",
      "127\n",
      "\n",
      "Example 9-1. app/models.py: role database model\n",
      "class Role(db.Model):\n",
      "    __tablename__ = 'roles'\n",
      "    id = db.Column(db.Integer, primary_key=True)\n",
      "    name = db.Column(db.String(64), unique=True)\n",
      "    default = db.Column(db.Boolean, default=False, index=True)\n",
      "    permissions = db.Column(db.Integer)\n",
      "    users = db.relationship('User', backref='role', lazy='dynamic')\n",
      "    def __init__(self, **kwargs):\n",
      "        super(Role, self).__init__(**kwargs)\n",
      "        if self.permissions is None:\n",
      "            self.permissions = 0\n",
      "The default field is one of the additions to this model. This field should be set to\n",
      "True for only one role and False for all the others. The role marked as default will be\n",
      "the one assigned to new users upon registration. Since the application is going to\n",
      "search the roles table to find the default one, this column is configured to have an\n",
      "index, as that will make searches much faster.\n",
      "Another addition to the model is the permissions field, which is an integer value that\n",
      "defines the list of permissions for the role in a compact way. Since SQLAlchemy will\n",
      "set this field to None by default, a class constructor is added that sets it to 0 if an initial\n",
      "value isn’t provided in the constructor arguments.\n",
      "The list of tasks for which permissions are needed is obviously application specific.\n",
      "For Flasky, the list is shown in Table 9-1.\n",
      "Table 9-1. Application permissions\n",
      "Task name\n",
      "Permission name Permission value\n",
      "Follow users\n",
      "FOLLOW\n",
      "1\n",
      "Comment on posts made by others\n",
      "COMMENT\n",
      "2\n",
      "Write articles\n",
      "WRITE\n",
      "4\n",
      "Moderate comments made by others\n",
      "MODERATE\n",
      "8\n",
      "Administration access\n",
      "ADMIN\n",
      "16\n",
      "The benefit of using powers of two for permission values is that it allows permissions\n",
      "to be combined, giving each possible combination of permissions a unique value to\n",
      "store in the role’s permissions field. For example, for a user role that gives users per‐\n",
      "mission to follow other users and comment on posts, the permission value is FOLLOW\n",
      "+ COMMENT = 3. This is a very efficient way to store the list of permissions assigned to\n",
      "each role.\n",
      "128 \n",
      "| \n",
      "Chapter 9: User Roles\n",
      "\n",
      "The code representation of Table 9-1 is shown in Example 9-2.\n",
      "Example 9-2. app/models.py: permission constants\n",
      "class Permission:\n",
      "    FOLLOW = 1\n",
      "    COMMENT = 2\n",
      "    WRITE = 4\n",
      "    MODERATE = 8\n",
      "    ADMIN = 16\n",
      "With the permission constants in place, a few new methods can be added to the Role\n",
      "model to manage permissions. These are shown in Example 9-3.\n",
      "Example 9-3. app/models.py: permission management in the Role model\n",
      "class Role(db.Model):\n",
      "    # ...\n",
      "    def add_permission(self, perm):\n",
      "        if not self.has_permission(perm):\n",
      "            self.permissions += perm\n",
      "    def remove_permission(self, perm):\n",
      "        if self.has_permission(perm):\n",
      "            self.permissions -= perm\n",
      "    def reset_permissions(self):\n",
      "        self.permissions = 0\n",
      "    def has_permission(self, perm):\n",
      "        return self.permissions & perm == perm\n",
      "The add_permission(), remove_permission(), and reset_permission() methods\n",
      "all use basic arithmetic operations to update the permission list. The\n",
      "has_permission() method is the most complex of the set, as it relies on the bitwise\n",
      "and operator & to check if a combined permission value includes the given basic per‐\n",
      "mission. You can play with these methods in a Python shell:\n",
      "(venv) $ flask shell\n",
      ">>> r = Role(name='User')\n",
      ">>> r.add_permission(Permission.FOLLOW)\n",
      ">>> r.add_permission(Permission.WRITE)\n",
      ">>> r.has_permission(Permission.FOLLOW)\n",
      "True\n",
      ">>> r.has_permission(Permission.ADMIN)\n",
      "False\n",
      ">>> r.reset_permissions()\n",
      "Database Representation of Roles \n",
      "| \n",
      "129\n",
      "\n",
      ">>> r.has_permission(Permission.FOLLOW)\n",
      "False\n",
      "Table 9-2 shows the list of user roles that will be supported in this application, along\n",
      "with the permission combinations that define each of them.\n",
      "Table 9-2. User roles\n",
      "User role\n",
      "Permissions\n",
      "Description\n",
      "None\n",
      "None\n",
      "Read-only access to the application. This applies to unknown users who are not\n",
      "logged in.\n",
      "User\n",
      "FOLLOW, COMMENT,\n",
      "WRITE\n",
      "Basic permissions to write articles and comments and to follow other users.\n",
      "This is the default for new users.\n",
      "Moderator\n",
      "FOLLOW, COMMENT,\n",
      "WRITE, MODERATE\n",
      "Adds permission to moderate comments made by other users.\n",
      "Administrator\n",
      "FOLLOW, COMMENT,\n",
      "WRITE, MODERATE,\n",
      "ADMIN\n",
      "Full access, which includes permission to change the roles of other users.\n",
      "Adding the roles to the database manually is time consuming and error prone, so\n",
      "instead a class method can be added to the Role class for this purpose, as shown in\n",
      "Example 9-4. This will make it easy to re-create the correct roles and permissions\n",
      "during unit testing and, more importantly, on the production server once the applica‐\n",
      "tion is deployed.\n",
      "Example 9-4. app/models.py: creating roles in the database\n",
      "class Role(db.Model):\n",
      "    # ...\n",
      "    @staticmethod\n",
      "    def insert_roles():\n",
      "        roles = {\n",
      "            'User': [Permission.FOLLOW, Permission.COMMENT, Permission.WRITE],\n",
      "            'Moderator': [Permission.FOLLOW, Permission.COMMENT,\n",
      "                          Permission.WRITE, Permission.MODERATE],\n",
      "            'Administrator': [Permission.FOLLOW, Permission.COMMENT,\n",
      "                              Permission.WRITE, Permission.MODERATE,\n",
      "                              Permission.ADMIN],\n",
      "        }\n",
      "        default_role = 'User'\n",
      "        for r in roles:\n",
      "            role = Role.query.filter_by(name=r).first()\n",
      "            if role is None:\n",
      "                role = Role(name=r)\n",
      "            role.reset_permissions()\n",
      "            for perm in roles[r]:\n",
      "                role.add_permission(perm)\n",
      "            role.default = (role.name == default_role)\n",
      "130 \n",
      "| \n",
      "Chapter 9: User Roles\n",
      "\n",
      "            db.session.add(role)\n",
      "        db.session.commit()\n",
      "The insert_roles() function does not directly create new role objects. Instead, it\n",
      "tries to find existing roles by name and update those. A new role object is created\n",
      "only for roles that aren’t in the database already. This is done so that the role list can\n",
      "be updated in the future when changes need to be made. To add a new role or change\n",
      "the permission assignments for a role, change the roles dictionary at the top of the\n",
      "function and then run the function again. Note that the \"Anonymous\" role does not\n",
      "need to be represented in the database, as it is the role that represents users who are\n",
      "not known and therefore are not in the database.\n",
      "Note also that insert_roles() is a static method, a special type of method that does\n",
      "not require an object to be created as it can be invoked directly on the class, for exam‐\n",
      "ple, as Role.insert_roles(). Static methods do not take a self argument like\n",
      "instance methods.\n",
      "Role Assignment\n",
      "When users register an account with the application, the correct role should be\n",
      "assigned to them. For most users, the role assigned at registration time will be the\n",
      "\"User\" role, as that is the role that is marked as a default. The only exception is made\n",
      "for the administrator, who needs to be assigned the \"Administrator\" role from the\n",
      "start. This user is identified by an email address stored in the FLASKY_ADMIN configu‐\n",
      "ration variable, so as soon as that email address appears in a registration request it\n",
      "can be given the correct role. Example 9-5 shows how this is done in the User model\n",
      "constructor.\n",
      "Example 9-5. app/models.py: defining a default role for users\n",
      "class User(UserMixin, db.Model):\n",
      "    # ...\n",
      "    def __init__(self, **kwargs):\n",
      "        super(User, self).__init__(**kwargs)\n",
      "        if self.role is None:\n",
      "            if self.email == current_app.config['FLASKY_ADMIN']:\n",
      "                self.role = Role.query.filter_by(name='Administrator').first()\n",
      "            if self.role is None:\n",
      "                self.role = Role.query.filter_by(default=True).first()\n",
      "    # ...\n",
      "The User constructor first invokes the constructors of the base classes, and if after\n",
      "that the object does not have a role defined, it sets the administrator or default role\n",
      "depending on the email address.\n",
      "Role Assignment \n",
      "| \n",
      "131\n",
      "\n",
      "Role Verification\n",
      "To simplify the implementation of roles and permissions, a helper method can be\n",
      "added to the User model that checks whether users have a given permission in the\n",
      "role they have been assigned. The implementation simply defers to the role methods\n",
      "added previously. This is shown in Example 9-6.\n",
      "Example 9-6. app/models.py: evaluating whether a user has a given permission\n",
      "from flask_login import UserMixin, AnonymousUserMixin\n",
      "class User(UserMixin, db.Model):\n",
      "    # ...\n",
      "    def can(self, perm):\n",
      "        return self.role is not None and self.role.has_permission(perm)\n",
      "    def is_administrator(self):\n",
      "        return self.can(Permission.ADMIN)\n",
      "class AnonymousUser(AnonymousUserMixin):\n",
      "    def can(self, permissions):\n",
      "        return False\n",
      "    def is_administrator(self):\n",
      "        return False\n",
      "login_manager.anonymous_user = AnonymousUser\n",
      "The can() method added to the User model returns True if the requested permission\n",
      "is present in the role, which means that the user should be allowed to perform the\n",
      "requested task. The check for administration permissions is so common that it is also\n",
      "implemented as a standalone is_administrator() method.\n",
      "For added convenience, a custom AnonymousUser class that implements the can()\n",
      "and is_administrator() methods is created as well. This will enable the application\n",
      "to freely call current_user.can() and current_user.is_administrator() without\n",
      "having to check whether the user is logged in first. Flask-Login is told to use the\n",
      "application’s \n",
      "custom \n",
      "anonymous \n",
      "user \n",
      "by \n",
      "setting \n",
      "its \n",
      "class \n",
      "in \n",
      "the\n",
      "login_manager.anonymous_user attribute.\n",
      "For cases in which an entire view function needs to be made available only to users\n",
      "with certain permissions, a custom decorator can be used. Example 9-7 shows the\n",
      "implementation of two decorators, one for generic permission checks and one that\n",
      "checks specifically for the administrator permission.\n",
      "132 \n",
      "| \n",
      "Chapter 9: User Roles\n",
      "\n",
      "Example 9-7. app/decorators.py: custom decorators that check user permissions\n",
      "from functools import wraps\n",
      "from flask import abort\n",
      "from flask_login import current_user\n",
      "from .models import Permission\n",
      "def permission_required(permission):\n",
      "    def decorator(f):\n",
      "        @wraps(f)\n",
      "        def decorated_function(*args, **kwargs):\n",
      "            if not current_user.can(permission):\n",
      "                abort(403)\n",
      "            return f(*args, **kwargs)\n",
      "        return decorated_function\n",
      "    return decorator\n",
      "def admin_required(f):\n",
      "    return permission_required(Permission.ADMIN)(f)\n",
      "These decorators are built with the help of the functools package from the Python\n",
      "standard library and return a 403 response, the “Forbidden” HTTP status code, when\n",
      "the current user does not have the requested permission. In Chapter 3, custom error\n",
      "pages were created for errors 404 and 500, so now a page for the 403 error is added in\n",
      "a similar way.\n",
      "The following are two examples that demonstrate the usage of these decorators:\n",
      "from .decorators import admin_required, permission_required\n",
      "@main.route('/admin')\n",
      "@login_required\n",
      "@admin_required\n",
      "def for_admins_only():\n",
      "    return \"For administrators!\"\n",
      "@main.route('/moderate')\n",
      "@login_required\n",
      "@permission_required(Permission.MODERATE)\n",
      "def for_moderators_only():\n",
      "    return \"For comment moderators!\"\n",
      "As a rule of thumb, the route decorator from Flask should be given first when using\n",
      "multiple decorators in a view function. The remaining decorators should be given in\n",
      "the order in which they need to evaluate when the view function is invoked. In these\n",
      "two cases, the user authenticated state needs to be checked first, since the user needs\n",
      "to be redirected to the login prompt if found to not be authenticated.\n",
      "Permissions may also need to be checked from templates, so the Permission class\n",
      "with all its constants needs to be accessible to them. To avoid having to add a tem‐\n",
      "Role Verification \n",
      "| \n",
      "133\n",
      "\n",
      "plate argument in every render_template() call, a context processor can be used.\n",
      "Context processors make variables available to all templates during rendering. This\n",
      "change is shown in Example 9-8.\n",
      "Example 9-8. app/main/__init__.py: adding the Permission class to the template context\n",
      "@main.app_context_processor\n",
      "def inject_permissions():\n",
      "    return dict(Permission=Permission)\n",
      "The new roles and permissions can be exercised in unit tests. Example 9-9 shows two\n",
      "of the tests. The source code on GitHub includes one for each role.\n",
      "Example 9-9. tests/test_user_model.py: unit tests for roles and permissions\n",
      "class UserModelTestCase(unittest.TestCase):\n",
      "    # ...\n",
      "    def test_user_role(self):\n",
      "        u = User(email='john@example.com', password='cat')\n",
      "        self.assertTrue(u.can(Permission.FOLLOW))\n",
      "        self.assertTrue(u.can(Permission.COMMENT))\n",
      "        self.assertTrue(u.can(Permission.WRITE))\n",
      "        self.assertFalse(u.can(Permission.MODERATE))\n",
      "        self.assertFalse(u.can(Permission.ADMIN))\n",
      "    def test_anonymous_user(self):\n",
      "        u = AnonymousUser()\n",
      "        self.assertFalse(u.can(Permission.FOLLOW))\n",
      "        self.assertFalse(u.can(Permission.COMMENT))\n",
      "        self.assertFalse(u.can(Permission.WRITE))\n",
      "        self.assertFalse(u.can(Permission.MODERATE))\n",
      "        self.assertFalse(u.can(Permission.ADMIN))\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 9a to check out this version of the applica‐\n",
      "tion. This update contains a database migration, so remember to\n",
      "run flask db upgrade after you check out the code.\n",
      "Before you move on to the next chapter, add the new roles to your development data‐\n",
      "base in a shell session:\n",
      "(venv) $ flask shell\n",
      ">>> Role.insert_roles()\n",
      ">>> Role.query.all()\n",
      "[<Role 'Administrator'>, <Role 'User'>, <Role 'Moderator'>]\n",
      "134 \n",
      "| \n",
      "Chapter 9: User Roles\n",
      "\n",
      "It is also a good idea to update the user list so that all the user accounts that were\n",
      "created before roles and permissions existed have a role assigned. You can run the fol‐\n",
      "lowing code in a Python shell to perform this update:\n",
      "(venv) $ flask shell\n",
      ">>> admin_role = Role.query.filter_by(name='Administrator').first()\n",
      ">>> default_role = Role.query.filter_by(default=True).first()\n",
      ">>> for u in User.query.all():\n",
      "...     if u.role is None:\n",
      "...         if u.email == app.config['FLASKY_ADMIN']:\n",
      "...             u.role = admin_role\n",
      "...         else:\n",
      "...             u.role = default_role\n",
      "...\n",
      ">>> db.session.commit()\n",
      "The user system is now fairly complete. The next chapter will make use of it to create\n",
      "user profile pages.\n",
      "Role Verification \n",
      "| \n",
      "135\n",
      "\n",
      "\n",
      "CHAPTER 10\n",
      "User Profiles\n",
      "In this chapter, user profiles for Flasky are implemented. All socially aware sites give\n",
      "their users a profile page, where a summary of the user’s participation in the website\n",
      "is presented. Users can advertise their presence on the website by sharing the URL to\n",
      "their profile page, so it is important that the URLs be short and easy to remember.\n",
      "Profile Information\n",
      "To make user profile pages more interesting, some additional information about\n",
      "users can be stored in the database. In Example 10-1 the User model is extended with\n",
      "several new fields.\n",
      "Example 10-1. app/models.py: user information fields\n",
      "class User(UserMixin, db.Model):\n",
      "    # ...\n",
      "    name = db.Column(db.String(64))\n",
      "    location = db.Column(db.String(64))\n",
      "    about_me = db.Column(db.Text())\n",
      "    member_since = db.Column(db.DateTime(), default=datetime.utcnow)\n",
      "    last_seen = db.Column(db.DateTime(), default=datetime.utcnow)\n",
      "The new fields store the user’s real name, location, self-written bio, date of registra‐\n",
      "tion, and date of last visit. The about_me field is assigned the type db.Text(). The\n",
      "difference between db.String and db.Text is that db.Text is a variable-length field\n",
      "and as such does not need a maximum length.\n",
      "The two timestamps are given a default value of the current time. Note that the\n",
      "datetime.utcnow is missing the () at the end. This is because the default argument\n",
      "in db.Column() can take a function as a value. Each time a default value needs to be\n",
      "137\n",
      "\n",
      "generated, SQLAlchemy invokes the function to produce it. This default value is all\n",
      "that is needed to manage the member_since field.\n",
      "The last_seen field is also initialized to the current time upon creation, but it needs\n",
      "to be refreshed each time the user accesses the site. A method in the User class can be\n",
      "added to perform this update. This is shown in Example 10-2.\n",
      "Example 10-2. app/models.py: refreshing a user’s last visit time\n",
      "class User(UserMixin, db.Model):\n",
      "    # ...\n",
      "    def ping(self):\n",
      "        self.last_seen = datetime.utcnow()\n",
      "        db.session.add(self)\n",
      "        db.session.commit()\n",
      "To keep the last visit date for all users updated, the ping() method must be called\n",
      "each time a request from a user is received. Because the before_app_request handler \n",
      "in the auth blueprint runs before every request, it can do this easily, as shown in\n",
      "Example 10-3.\n",
      "Example 10-3. app/auth/views.py: pinging the logged-in user\n",
      "@auth.before_app_request\n",
      "def before_request():\n",
      "    if current_user.is_authenticated:\n",
      "        current_user.ping()\n",
      "        if not current_user.confirmed \\\n",
      "                and request.endpoint \\\n",
      "                and request.blueprint != 'auth' \\\n",
      "                and request.endpoint != 'static':\n",
      "            return redirect(url_for('auth.unconfirmed'))\n",
      "User Profile Page\n",
      "Creating a profile page for each user does not present any new challenges.\n",
      "Example 10-4 shows the route definition.\n",
      "Example 10-4. app/main/views.py: profile page route\n",
      "@main.route('/user/<username>')\n",
      "def user(username):\n",
      "    user = User.query.filter_by(username=username).first_or_404()\n",
      "    return render_template('user.html', user=user)\n",
      "138 \n",
      "| \n",
      "Chapter 10: User Profiles\n",
      "\n",
      "This route is added in the main blueprint. For a user named john, the profile page will\n",
      "be at http://localhost:5000/user/john. The username given in the URL is searched in\n",
      "the database and, if found, the user.html template is rendered with it as the argument.\n",
      "An invalid username sent into this route will cause a 404 error to be returned. With\n",
      "Flask-SQLAlchemy, the search and error cases can be nicely combined in a single\n",
      "statement using the first_or_404() method of the query object. The user.html tem‐\n",
      "plate is going to need to present user information, so it receives the user object as an\n",
      "argument. An initial version of this template is shown in Example 10-5.\n",
      "Example 10-5. app/templates/user.html: user profile template\n",
      "{% extends \"base.html\" %}\n",
      "{% block title %}Flasky - {{ user.username }}{% endblock %}\n",
      "{% block page_content %}\n",
      "<div class=\"page-header\">\n",
      "    <h1>{{ user.username }}</h1>\n",
      "    {% if user.name or user.location %}\n",
      "    <p>\n",
      "        {% if user.name %}{{ user.name }}{% endif %}\n",
      "        {% if user.location %}\n",
      "            From <a href=\"http://maps.google.com/?q={{ user.location }}\">\n",
      "                {{ user.location }}\n",
      "            </a>\n",
      "        {% endif %}\n",
      "    </p>\n",
      "    {% endif %}\n",
      "    {% if current_user.is_administrator() %}\n",
      "    <p><a href=\"mailto:{{ user.email }}\">{{ user.email }}</a></p>\n",
      "    {% endif %}\n",
      "    {% if user.about_me %}<p>{{ user.about_me }}</p>{% endif %}\n",
      "    <p>\n",
      "        Member since {{ moment(user.member_since).format('L') }}.\n",
      "        Last seen {{ moment(user.last_seen).fromNow() }}.\n",
      "    </p>\n",
      "</div>\n",
      "{% endblock %}\n",
      "This template has a few interesting implementation details:\n",
      "• The name and location fields are rendered inside a single <p> element. A Jinja2\n",
      "conditional ensures that the <p> element is created only when at least one of the\n",
      "fields is defined.\n",
      "• The user location field is rendered as a link to a Google Maps query, so that\n",
      "clicking on it opens a map centered on the location.\n",
      "User Profile Page \n",
      "| \n",
      "139\n",
      "\n",
      "• If the logged-in user is an administrator, then the email address of the user is\n",
      "shown, rendered as a mailto link. This is useful when an administrator is viewing\n",
      "the profile page of another user and needs to contact the user.\n",
      "• The two timestamps for the user are rendered to the page using Flask-Moment,\n",
      "as shown in Chapter 3.\n",
      "As most users will want easy access to their own profile page, a link to it can be added\n",
      "to the navigation bar. The relevant changes to the base.html template are shown in\n",
      "Example 10-6.\n",
      "Example 10-6. app/templates/base.html: add link to profile page in the navigation bar\n",
      "{% if current_user.is_authenticated %}\n",
      "<li>\n",
      "    <a href=\"{{ url_for('main.user', username=current_user.username) }}\">\n",
      "        Profile\n",
      "    </a>\n",
      "</li>\n",
      "{% endif %}\n",
      "Using a conditional for the profile page link is necessary because the navigation bar is\n",
      "also rendered for unauthenticated users, in which case the profile link is skipped.\n",
      "Figure 10-1 shows how the profile page looks in the browser. The new profile link in\n",
      "the navigation bar is also shown.\n",
      "Figure 10-1. User profile page\n",
      "140 \n",
      "| \n",
      "Chapter 10: User Profiles\n",
      "\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 10a to check out this version of the applica‐\n",
      "tion. This update contains a database migration, so remember to\n",
      "run flask db upgrade after you check out the code.\n",
      "Profile Editor\n",
      "There are two different use cases related to editing of user profiles. The most obvious\n",
      "is that users need to have access to a page where they can enter information about\n",
      "themselves to present in their profile pages. A less obvious but also important\n",
      "requirement is to let administrators edit the profiles of other users—not only their\n",
      "personal information items but also other fields in the User model to which users\n",
      "have no direct access, such as the user role. Because the two profile editing require‐\n",
      "ments are substantially different, two different forms will be created.\n",
      "User-Level Profile Editor\n",
      "The profile editing form for regular users is shown in Example 10-7.\n",
      "Example 10-7. app/main/forms.py: edit profile form\n",
      "class EditProfileForm(FlaskForm):\n",
      "    name = StringField('Real name', validators=[Length(0, 64)])\n",
      "    location = StringField('Location', validators=[Length(0, 64)])\n",
      "    about_me = TextAreaField('About me')\n",
      "    submit = SubmitField('Submit')\n",
      "Note that as all the fields in this form are optional, the length validator allows a length\n",
      "of zero as a minimum. The route definition that uses this form is shown in\n",
      "Example 10-8.\n",
      "Example 10-8. app/main/views.py: edit profile route\n",
      "@main.route('/edit-profile', methods=['GET', 'POST'])\n",
      "@login_required\n",
      "def edit_profile():\n",
      "    form = EditProfileForm()\n",
      "    if form.validate_on_submit():\n",
      "        current_user.name = form.name.data\n",
      "        current_user.location = form.location.data\n",
      "        current_user.about_me = form.about_me.data\n",
      "        db.session.add(current_user._get_current_object())\n",
      "        db.session.commit()\n",
      "        flash('Your profile has been updated.')\n",
      "        return redirect(url_for('.user', username=current_user.username))\n",
      "    form.name.data = current_user.name\n",
      "Profile Editor \n",
      "| \n",
      "141\n",
      "\n",
      "    form.location.data = current_user.location\n",
      "    form.about_me.data = current_user.about_me\n",
      "    return render_template('edit_profile.html', form=form)\n",
      "As in previous forms, the data associated with each form field is available at\n",
      "form.<field-name>.data. This is useful not only to obtain values submitted by the\n",
      "user, but also to provide initial values that are shown to the user for editing. When\n",
      "form.validate_on_submit() is False, the three fields in this form are initialized\n",
      "from the corresponding fields in current_user. Then, when the form is submitted,\n",
      "the data attributes of the form fields contain the updated values, so these are moved\n",
      "back into the fields of the user object before the object is saved back to the database.\n",
      "Figure 10-2 shows the profile editing page.\n",
      "Figure 10-2. Profile editor\n",
      "To make it easy for users to reach this page, a direct link can be added in the profile\n",
      "page, as shown in Example 10-9.\n",
      "142 \n",
      "| \n",
      "Chapter 10: User Profiles\n",
      "\n",
      "Example 10-9. app/templates/user.html: edit profile link\n",
      "{% if user == current_user %}\n",
      "<a class=\"btn btn-default\" href=\"{{ url_for('.edit_profile') }}\">\n",
      "    Edit Profile\n",
      "</a>\n",
      "{% endif %}\n",
      "The conditional that encloses the link will make the link appear only when users are\n",
      "viewing their own profiles.\n",
      "Administrator-Level Profile Editor\n",
      "The profile editing form for administrators is more complex than the one for regular\n",
      "users. In addition to the three profile information fields, this form allows administra‐\n",
      "tors to edit a user’s email, username, confirmed status, and role. The form is shown in\n",
      "Example 10-10.\n",
      "Example 10-10. app/main/forms.py: profile editing form for administrators\n",
      "class EditProfileAdminForm(FlaskForm):\n",
      "    email = StringField('Email', validators=[DataRequired(), Length(1, 64),\n",
      "                                             Email()])\n",
      "    username = StringField('Username', validators=[\n",
      "        DataRequired(), Length(1, 64),\n",
      "        Regexp('^[A-Za-z][A-Za-z0-9_.]*$', 0,\n",
      "               'Usernames must have only letters, numbers, dots or '\n",
      "               'underscores')])\n",
      "    confirmed = BooleanField('Confirmed')\n",
      "    role = SelectField('Role', coerce=int)\n",
      "    name = StringField('Real name', validators=[Length(0, 64)])\n",
      "    location = StringField('Location', validators=[Length(0, 64)])\n",
      "    about_me = TextAreaField('About me')\n",
      "    submit = SubmitField('Submit')\n",
      "    def __init__(self, user, *args, **kwargs):\n",
      "        super(EditProfileAdminForm, self).__init__(*args, **kwargs)\n",
      "        self.role.choices = [(role.id, role.name)\n",
      "                             for role in Role.query.order_by(Role.name).all()]\n",
      "        self.user = user\n",
      "    def validate_email(self, field):\n",
      "        if field.data != self.user.email and \\\n",
      "                User.query.filter_by(email=field.data).first():\n",
      "            raise ValidationError('Email already registered.')\n",
      "    def validate_username(self, field):\n",
      "        if field.data != self.user.username and \\\n",
      "Profile Editor \n",
      "| \n",
      "143\n",
      "\n",
      "                User.query.filter_by(username=field.data).first():\n",
      "            raise ValidationError('Username already in use.')\n",
      "The SelectField is WTForm’s wrapper for the <select> HTML form control, which\n",
      "implements a drop-down list, used in this form to select a user role. An instance of\n",
      "SelectField must have the items set in its choices attribute. They must be given as a\n",
      "list of tuples, with each tuple consisting of two values: an identifier for the item and\n",
      "the text to show in the control as a string. The choices list is set in the form’s con‐\n",
      "structor, with values obtained from the Role model with a query that sorts all the\n",
      "roles alphabetically by name. The identifier for each tuple is set to the id of each role,\n",
      "and since these are integers, a coerce=int argument is added to the SelectField\n",
      "constructor so that the field values are stored as integers instead of the default, which\n",
      "is strings.\n",
      "The email and username fields are constructed in the same way as in the authentica‐\n",
      "tion forms, but their validation requires some careful handling. The validation condi‐\n",
      "tion used for both these fields must first check whether a change to the field was\n",
      "made, and only when there is a change should it ensure that the new value does not\n",
      "duplicate another user’s. When these fields are not changed, then validation should\n",
      "pass. To implement this logic, the form’s constructor receives the user object as an\n",
      "argument and saves it as a member variable, which is later used in the custom valida‐\n",
      "tion methods.\n",
      "The route definition for the administrator’s profile editor is shown in Example 10-11.\n",
      "Example 10-11. app/main/views.py: edit profile route for administrators\n",
      "from ..decorators import admin_required\n",
      "@main.route('/edit-profile/<int:id>', methods=['GET', 'POST'])\n",
      "@login_required\n",
      "@admin_required\n",
      "def edit_profile_admin(id):\n",
      "    user = User.query.get_or_404(id)\n",
      "    form = EditProfileAdminForm(user=user)\n",
      "    if form.validate_on_submit():\n",
      "        user.email = form.email.data\n",
      "        user.username = form.username.data\n",
      "        user.confirmed = form.confirmed.data\n",
      "        user.role = Role.query.get(form.role.data)\n",
      "        user.name = form.name.data\n",
      "        user.location = form.location.data\n",
      "        user.about_me = form.about_me.data\n",
      "        db.session.add(user)\n",
      "        db.session.commit()\n",
      "        flash('The profile has been updated.')\n",
      "        return redirect(url_for('.user', username=user.username))\n",
      "    form.email.data = user.email\n",
      "144 \n",
      "| \n",
      "Chapter 10: User Profiles\n",
      "\n",
      "    form.username.data = user.username\n",
      "    form.confirmed.data = user.confirmed\n",
      "    form.role.data = user.role_id\n",
      "    form.name.data = user.name\n",
      "    form.location.data = user.location\n",
      "    form.about_me.data = user.about_me\n",
      "    return render_template('edit_profile.html', form=form, user=user)\n",
      "This route has largely the same structure as the simpler one for regular users, but it\n",
      "includes the admin_required decorator created in Chapter 9, which will automati‐\n",
      "cally return a 403 error for any users who are not administrators that try to use this\n",
      "route.\n",
      "The user id is given as a dynamic argument in the URL, so Flask-SQLAlchemy’s\n",
      "get_or_404() convenience function can be used, knowing that if the id is invalid the\n",
      "request will return a code 404 error. The SelectField used for the user role also\n",
      "deserves to be studied. When setting the initial value for the field, the role_id is\n",
      "assigned to field.role.data because the list of tuples set in the choices attribute\n",
      "uses the numeric identifiers to reference each option. When the form is submitted,\n",
      "the id is extracted from the field’s data attribute and used in a query to load the\n",
      "selected role object by its id once again. The coerce=int argument used in the\n",
      "SelectField declaration in the form ensures that the data attribute of this field is\n",
      "always converted to an integer.\n",
      "To link to this page, another button is added in the user profile page, as shown in\n",
      "Example 10-12.\n",
      "Example 10-12. app/templates/user.html: profile editing link for administrators\n",
      "{% if current_user.is_administrator() %}\n",
      "<a class=\"btn btn-danger\"\n",
      "        href=\"{{ url_for('.edit_profile_admin', id=user.id) }}\">\n",
      "    Edit Profile [Admin]\n",
      "</a>\n",
      "{% endif %}\n",
      "This button is rendered with a different Bootstrap style to call attention to it. The\n",
      "conditional that wraps it makes the button appear in profile pages only if the logged-\n",
      "in user has the administrator role.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 10b to check out this version of the applica‐\n",
      "tion.\n",
      "Profile Editor \n",
      "| \n",
      "145\n",
      "\n",
      "User Avatars\n",
      "The look of the profile pages can be improved by showing avatar pictures of users. In\n",
      "this section, you will learn how to add user avatars provided by Gravatar, the leading\n",
      "avatar service. Gravatar associates avatar images with email addresses. Users create an\n",
      "account at https://gravatar.com and then upload their images. The service exposes the\n",
      "user’s avatar through a specially crafted URL that includes the MD5 hash of the user’s\n",
      "email address, which can be calculated as follows:\n",
      "(venv) $ python\n",
      ">>> import hashlib\n",
      ">>> hashlib.md5('john@example.com'.encode('utf-8')).hexdigest()\n",
      "'d4c74594d841139328695756648b6bd6'\n",
      "The avatar URLs are then generated by appending the MD5 hash to the https://\n",
      "secure.gravatar.com/avatar/ URL. For example, you can type https://secure.grava‐\n",
      "tar.com/avatar/d4c74594d841139328695756648b6bd6 in your browser’s address bar\n",
      "to get the avatar image for the email address john@example.com, or a default avatar\n",
      "image if that email address does not have an avatar registered. After you build the\n",
      "basic avatar URL, a few query string arguments can be used to configure the charac‐\n",
      "teristics of the avatar image, as described in Table 10-1.\n",
      "Table 10-1. Gravatar query string arguments\n",
      "Argument\n",
      "name\n",
      "Description\n",
      "s\n",
      "Image size, in pixels.\n",
      "r\n",
      "Image rating. Options are \"g\", \"pg\", \"r\", and \"x\".\n",
      "d\n",
      "The default image generator for users who have no avatars registered with the Gravatar service. Options\n",
      "are \"404\" to return a 404 error, a URL that points to a default image, or one of the following image\n",
      "generators: \"mm\", \"identicon\", \"monsterid\", \"wavatar\", \"retro\", or \"blank\".\n",
      "fd\n",
      "Force the use of default avatars.\n",
      "For example, adding ?d=identicon to the avatar URL for john@example.com will gen‐\n",
      "erate a different default avatar that is based on geometric designs. All these options to\n",
      "generate avatar URLs can be added to the User model. The implementation is shown\n",
      "in Example 10-13.\n",
      "Example 10-13. app/models.py: gravatar URL generation\n",
      "import hashlib\n",
      "from flask import request\n",
      "class User(UserMixin, db.Model):\n",
      "    # ...\n",
      "    def gravatar(self, size=100, default='identicon', rating='g'):\n",
      "146 \n",
      "| \n",
      "Chapter 10: User Profiles\n",
      "\n",
      "        url = 'https://secure.gravatar.com/avatar'\n",
      "        hash = hashlib.md5(self.email.lower().encode('utf-8')).hexdigest()\n",
      "        return '{url}/{hash}?s={size}&d={default}&r={rating}'.format(\n",
      "            url=url, hash=hash, size=size, default=default, rating=rating)\n",
      "The avatar URL is generated from the base URL, the MD5 hash of the user’s email\n",
      "address, and the arguments, all of which have default values. Note that one of the\n",
      "requirements of the Gravatar service is that the email address from which the MD5\n",
      "hash is obtained is normalized to contain only lowercase alphabetical characters, so\n",
      "that conversion is also added to this method. With this implementation it is easy to\n",
      "generate avatar URLs in the Python shell:\n",
      "(venv) $ flask shell\n",
      ">>> u = User(email='john@example.com')\n",
      ">>> u.gravatar()\n",
      "'https://secure.gravatar.com/avatar/d4c74594d841139328695756648b6bd6?s=100&d=\n",
      "identicon&r=g'\n",
      ">>> u.gravatar(size=256)\n",
      "'https://secure.gravatar.com/avatar/d4c74594d841139328695756648b6bd6?s=256&d=\n",
      "identicon&r=g'\n",
      "The gravatar() method can also be invoked from Jinja2 templates. Example 10-14\n",
      "shows how a 256-pixel avatar can be added to the profile page.\n",
      "Example 10-14. app/templates/user.html: adding an avatar to the profile page\n",
      "...\n",
      "<img class=\"img-rounded profile-thumbnail\" src=\"{{ user.gravatar(size=256) }}\">\n",
      "<div class=\"profile-header\">\n",
      "   ...\n",
      "</div>\n",
      "...\n",
      "The profile-thumbnail CSS class helps with the positioning of the image on the\n",
      "page. The <div> element that follows the image encapsulates the profile information\n",
      "and uses the profile-header CSS class to improve the formatting. You can see the\n",
      "definition of the CSS class in the GitHub repository for the application.\n",
      "Using a similar approach, the base template adds a small thumbnail image of the\n",
      "logged-in user in the navigation bar. To better format the avatar pictures in the page,\n",
      "custom CSS classes are used. You can find these in the source code repository in a\n",
      "styles.css file added to the application’s static file folder and referenced from the\n",
      "base.html template. Figure 10-3 shows the user profile page with avatar.\n",
      "User Avatars \n",
      "| \n",
      "147\n",
      "\n",
      "Figure 10-3. User profile page with avatar\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 10c to check out this version of the applica‐\n",
      "tion.\n",
      "The generation of avatars requires an MD5 hash to be generated, which is a CPU-\n",
      "intensive operation. If a large number of avatars need to be generated for a page, then\n",
      "the computational work can add up and become significant. Since the MD5 hash for\n",
      "a user will remain constant for as long as the email address stays the same, it can be\n",
      "cached in the User model. Example 10-15 shows the changes to the User model to\n",
      "store the MD5 hashes in the database.\n",
      "Example 10-15. app/models.py: gravatar URL generation with caching of MD5 hashes\n",
      "class User(UserMixin, db.Model):\n",
      "    # ...\n",
      "    avatar_hash = db.Column(db.String(32))\n",
      "    def __init__(self, **kwargs):\n",
      "        # ...\n",
      "        if self.email is not None and self.avatar_hash is None:\n",
      "148 \n",
      "| \n",
      "Chapter 10: User Profiles\n",
      "\n",
      "            self.avatar_hash = self.gravatar_hash()\n",
      "    def change_email(self, token):\n",
      "        # ...\n",
      "        self.email = new_email\n",
      "        self.avatar_hash = self.gravatar_hash()\n",
      "        db.session.add(self)\n",
      "        return True\n",
      "    def gravatar_hash(self):\n",
      "        return hashlib.md5(self.email.lower().encode('utf-8')).hexdigest()\n",
      "    def gravatar(self, size=100, default='identicon', rating='g'):\n",
      "        if request.is_secure:\n",
      "            url = 'https://secure.gravatar.com/avatar'\n",
      "        else:\n",
      "            url = 'http://www.gravatar.com/avatar'\n",
      "        hash = self.avatar_hash or self.gravatar_hash()\n",
      "        return '{url}/{hash}?s={size}&d={default}&r={rating}'.format(\n",
      "            url=url, hash=hash, size=size, default=default, rating=rating)\n",
      "To avoid duplicating the logic to compute the gravatar hash, a new method,\n",
      "gravatar_hash(), is added that performs this task. During model initialization, the\n",
      "hash is stored in the new avatar_hash model column. If the user updates the email\n",
      "address, then the hash is recalculated. The gravatar() method uses the stored hash if\n",
      "available, and if not, it generates a new hash as before.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 10d to check out this version of the applica‐\n",
      "tion. This update contains a database migration, so remember to\n",
      "run flask db upgrade after you check out the code.\n",
      "In the next chapter, the blogging engine that powers this application will be created.\n",
      "User Avatars \n",
      "| \n",
      "149\n",
      "\n",
      "\n",
      "CHAPTER 11\n",
      "Blog Posts\n",
      "This chapter is dedicated to the implementation of Flasky’s main feature, which is to\n",
      "allow users to read and write blog posts. Here you will learn a few new techniques for\n",
      "reuse of templates, pagination of long lists of items, and working with rich text.\n",
      "Blog Post Submission and Display\n",
      "To support blog posts, a new database model that represents them is necessary. This\n",
      "model is shown in Example 11-1.\n",
      "Example 11-1. app/models.py: Post model\n",
      "class Post(db.Model):\n",
      "    __tablename__ = 'posts'\n",
      "    id = db.Column(db.Integer, primary_key=True)\n",
      "    body = db.Column(db.Text)\n",
      "    timestamp = db.Column(db.DateTime, index=True, default=datetime.utcnow)\n",
      "    author_id = db.Column(db.Integer, db.ForeignKey('users.id'))\n",
      "class User(UserMixin, db.Model):\n",
      "    # ...\n",
      "    posts = db.relationship('Post', backref='author', lazy='dynamic')\n",
      "A blog post is represented by a body, a timestamp, and a one-to-many relationship\n",
      "from the User model. The body field is defined with type db.Text so that there is no\n",
      "limitation on the length.\n",
      "The form that will be shown in the main page of the application lets users write a blog\n",
      "post. This form is very simple; it contains just a text area where the blog post can be\n",
      "typed and a submit button. The form definition is shown in Example 11-2.\n",
      "151\n",
      "\n",
      "Example 11-2. app/main/forms.py: blog post form\n",
      "class PostForm(FlaskForm):\n",
      "    body = TextAreaField(\"What's on your mind?\", validators=[DataRequired()])\n",
      "    submit = SubmitField('Submit')\n",
      "The index() view function handles the form and passes the list of old blog posts to\n",
      "the template, as shown in Example 11-3.\n",
      "Example 11-3. app/main/views.py: home page route with a blog post\n",
      "@main.route('/', methods=['GET', 'POST'])\n",
      "def index():\n",
      "    form = PostForm()\n",
      "    if current_user.can(Permission.WRITE_ARTICLES) and form.validate_on_submit():\n",
      "        post = Post(body=form.body.data,\n",
      "                    author=current_user._get_current_object())\n",
      "        db.session.add(post)\n",
      "        db.session.commit()\n",
      "        return redirect(url_for('.index'))\n",
      "    posts = Post.query.order_by(Post.timestamp.desc()).all()\n",
      "    return render_template('index.html', form=form, posts=posts)\n",
      "This view function passes the form and the complete list of blog posts to the template.\n",
      "The list of posts is ordered by timestamp, in descending order. The blog post form is\n",
      "handled in the usual manner, with the creation of a new Post instance when a valid\n",
      "submission is received. The current user’s permission to write articles is checked\n",
      "before allowing the new post.\n",
      "Note how the author attribute of the new post object is set to the expression\n",
      "current_user._get_current_object(). The current_user variable from Flask-\n",
      "Login, like all context variables, is implemented as a thread-local proxy object. This\n",
      "object behaves like a user object but is really a thin wrapper that contains the actual\n",
      "user object inside. The database needs a real user object, which is obtained by calling\n",
      "_get_current_object() on the proxy object.\n",
      "The form is rendered below the greeting in the index.html template, followed by the\n",
      "blog posts. The list of blog posts is a first attempt to create a blog post timeline, with\n",
      "all the blog posts in the database listed in chronological order from newest to oldest.\n",
      "The changes to the template are shown in Example 11-4.\n",
      "152 \n",
      "| \n",
      "Chapter 11: Blog Posts\n",
      "\n",
      "Example 11-4. app/templates/index.html: home page template with blog posts\n",
      "{% extends \"base.html\" %}\n",
      "{% import \"bootstrap/wtf.html\" as wtf %}\n",
      "...\n",
      "<div>\n",
      "    {% if current_user.can(Permission.WRITE_ARTICLES) %}\n",
      "    {{ wtf.quick_form(form) }}\n",
      "    {% endif %}\n",
      "</div>\n",
      "<ul class=\"posts\">\n",
      "    {% for post in posts %}\n",
      "    <li class=\"post\">\n",
      "        <div class=\"profile-thumbnail\">\n",
      "            <a href=\"{{ url_for('.user', username=post.author.username) }}\">\n",
      "                <img class=\"img-rounded profile-thumbnail\"\n",
      "                    src=\"{{ post.author.gravatar(size=40) }}\">\n",
      "            </a>\n",
      "        </div>\n",
      "        <div class=\"post-date\">{{ moment(post.timestamp).fromNow() }}</div>\n",
      "        <div class=\"post-author\">\n",
      "            <a href=\"{{ url_for('.user', username=post.author.username) }}\">\n",
      "                {{ post.author.username }}\n",
      "            </a>\n",
      "        </div>\n",
      "        <div class=\"post-body\">{{ post.body }}</div>\n",
      "    </li>\n",
      "    {% endfor %}\n",
      "</ul>\n",
      "...\n",
      "Note that the User.can() method is used to skip the blog post form for users who do\n",
      "not have the WRITE permission in their role. The blog post list is implemented as an\n",
      "HTML unordered list, with CSS classes giving it a nicer formatting. A small avatar of\n",
      "the author is rendered on the left side, and both the avatar and the author’s username\n",
      "are rendered as links to the user’s profile page. The CSS styles used are stored in the\n",
      "styles.css file located in the application’s static directory. You can review this file in the\n",
      "GitHub repository. Figure 11-1 shows the home page with submission form and blog\n",
      "post list.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 11a to check out this version of the applica‐\n",
      "tion. This update contains a database migration, so remember to\n",
      "run flask db upgrade after you check out the code.\n",
      "Blog Post Submission and Display \n",
      "| \n",
      "153\n",
      "\n",
      "Figure 11-1. Home page with blog submission form and blog post list\n",
      "Blog Posts on Profile Pages\n",
      "The user profile page can be improved by showing a list of blog posts authored by the\n",
      "user. Example 11-5 shows the changes to the view function to obtain the post list.\n",
      "Example 11-5. app/main/views.py: profile page route with blog posts\n",
      "@main.route('/user/<username>')\n",
      "def user(username):\n",
      "    user = User.query.filter_by(username=username).first()\n",
      "    if user is None:\n",
      "        abort(404)\n",
      "    posts = user.posts.order_by(Post.timestamp.desc()).all()\n",
      "    return render_template('user.html', user=user, posts=posts)\n",
      "The list of blog posts for a user is obtained from the User.posts relationship. This\n",
      "works like a query object, so filters such as order_by() can be used on it like in a\n",
      "regular query object.\n",
      "154 \n",
      "| \n",
      "Chapter 11: Blog Posts\n",
      "\n",
      "The user.html template needs to have the same <ul> HTML tree that renders a list of\n",
      "blog posts in index.html, but having to maintain two identical copies of a piece of\n",
      "HTML code is not ideal. For cases like this, Jinja2’s include directive is very useful.\n",
      "The snippet of HTML that generates the post list can be moved to a separate file that\n",
      "both index.html and user.html can include. Example 11-6 shows how this include\n",
      "looks in user.html.\n",
      "Example 11-6. app/templates/user.html: profile page template with blog posts\n",
      "...\n",
      "<h3>Posts by {{ user.username }}</h3>\n",
      "{% include '_posts.html' %}\n",
      "...\n",
      "To complete this reorganization, the <ul> tree from index.html is moved to the new\n",
      "template _posts.html, and replaced with another include directive like the one just\n",
      "shown. Note that the use of an underscore prefix in the _posts.html template name is\n",
      "not a requirement; this is merely a convention to distinguish full and partial tem‐\n",
      "plates.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 11b to check out this version of the applica‐\n",
      "tion.\n",
      "Paginating Long Blog Post Lists\n",
      "As the site grows and the number of blog posts increases, it will become slow and\n",
      "impractical to show the complete list of posts on the home and profile pages. Big\n",
      "pages take longer to generate, download, and render in the web browser, so the qual‐\n",
      "ity of the user experience decreases as the pages get larger. The solution is to paginate\n",
      "the data and render it in chunks.\n",
      "Creating Fake Blog Post Data\n",
      "To be able to work with multiple pages of blog posts, it is necessary to have a test\n",
      "database with a large volume of data. Manually adding new database entries is time\n",
      "consuming and tedious; an automated solution is more appropriate. There are several\n",
      "Python packages that can be used to generate fake information. A fairly complete one\n",
      "is Faker, which is installed with pip:\n",
      "(venv) $ pip install faker\n",
      "Paginating Long Blog Post Lists \n",
      "| \n",
      "155\n",
      "\n",
      "The Faker package is not, strictly speaking, a dependency of the application, because\n",
      "it is needed only during development. To separate the production dependencies from\n",
      "the development dependencies, the requirements.txt file can be replaced with a\n",
      "requirements subdirectory that stores different sets of dependencies. Inside this new\n",
      "subdirectory, a dev.txt file can list the dependencies that are necessary for develop‐\n",
      "ment and a prod.txt file can list the dependencies that are needed in production. As\n",
      "there are a large number of dependencies that will be in both lists, a common.txt file is\n",
      "added for those, and then the dev.txt and prod.txt lists use the -r prefix to include it.\n",
      "Example 11-7 shows the dev.txt file.\n",
      "Example 11-7. requirements/dev.txt: development requirements file\n",
      "-r common.txt\n",
      "faker==0.7.18\n",
      "Example 11-8 shows a new module added to the application that contains two func‐\n",
      "tions that generate fake users and posts.\n",
      "Example 11-8. app/fake.py: generating fake users and blog posts\n",
      "from random import randint\n",
      "from sqlalchemy.exc import IntegrityError\n",
      "from faker import Faker\n",
      "from . import db\n",
      "from .models import User, Post\n",
      "def users(count=100):\n",
      "    fake = Faker()\n",
      "    i = 0\n",
      "    while i < count:\n",
      "        u = User(email=fake.email(),\n",
      "                 username=fake.user_name(),\n",
      "                 password='password',\n",
      "                 confirmed=True,\n",
      "                 name=fake.name(),\n",
      "                 location=fake.city(),\n",
      "                 about_me=fake.text(),\n",
      "                 member_since=fake.past_date())\n",
      "        db.session.add(u)\n",
      "        try:\n",
      "            db.session.commit()\n",
      "            i += 1\n",
      "        except IntegrityError:\n",
      "            db.session.rollback()\n",
      "def posts(count=100):\n",
      "    fake = Faker()\n",
      "    user_count = User.query.count()\n",
      "156 \n",
      "| \n",
      "Chapter 11: Blog Posts\n",
      "\n",
      "    for i in range(count):\n",
      "        u = User.query.offset(randint(0, user_count - 1)).first()\n",
      "        p = Post(body=fake.text(),\n",
      "                 timestamp=fake.past_date(),\n",
      "                 author=u)\n",
      "        db.session.add(p)\n",
      "    db.session.commit()\n",
      "The attributes of these fake objects are produced by random information generators\n",
      "provided by the Faker package, which can generate real-looking names, emails, sen‐\n",
      "tences, and many more attributes.\n",
      "The email addresses and usernames of users must be unique, but since Faker gener‐\n",
      "ates these in a completely random fashion, there is a risk of having duplicates. In the\n",
      "unlikely event that a duplicate is generated, the database session commit will throw\n",
      "an IntegrityError exception. The exception is handled by rolling back the session to\n",
      "cancel that duplicate user. The loop will run until the requested number of unique\n",
      "users are generated.\n",
      "The random post generation must assign a random user to each post. For this, the\n",
      "offset() query filter is used. This filter discards the number of results given as an\n",
      "argument. By setting a random offset and then calling first(), a different random\n",
      "user is obtained each time.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 11c to check out this version of the applica‐\n",
      "tion. To ensure that you have all the dependencies installed, also\n",
      "run pip install -r requirements/dev.txt.\n",
      "The new functions make it easy to create a large number of fake users and posts from\n",
      "the Python shell:\n",
      "(venv) $ flask shell\n",
      ">>> from app import fake\n",
      ">>> fake.users(100)\n",
      ">>> fake.posts(100)\n",
      "If you run the application now, you will see a long list of random blog posts on the\n",
      "home page, by many different users.\n",
      "Rendering in Pages\n",
      "Example 11-9 shows the changes to the home page route to support pagination.\n",
      "Paginating Long Blog Post Lists \n",
      "| \n",
      "157\n",
      "\n",
      "Example 11-9. app/main/views.py: paginating the blog post list\n",
      "@main.route('/', methods=['GET', 'POST'])\n",
      "def index():\n",
      "    # ...\n",
      "    page = request.args.get('page', 1, type=int)\n",
      "    pagination = Post.query.order_by(Post.timestamp.desc()).paginate(\n",
      "        page, per_page=current_app.config['FLASKY_POSTS_PER_PAGE'],\n",
      "        error_out=False)\n",
      "    posts = pagination.items\n",
      "    return render_template('index.html', form=form, posts=posts,\n",
      "                           pagination=pagination)\n",
      "The page number to render is obtained from the request’s query string, which is avail‐\n",
      "able as request.args. When a page isn’t given, a default page of 1 (the first page) is\n",
      "used. The type=int argument ensures that if the argument cannot be converted to an\n",
      "integer, the default value is returned.\n",
      "To load a single page of records, the final call to the all() method of the query object\n",
      "is replaced with Flask-SQLAlchemy’s paginate(). The paginate() method takes the\n",
      "page number as its first and only required argument. An optional per_page argument\n",
      "can be given to indicate the size of each page, in number of items. If this argument is\n",
      "not specified, the default is 20 items per page. Another optional argument called\n",
      "error_out can be set to True (the default) to issue a code 404 error when a page out‐\n",
      "side of the valid range is requested. If error_out is False, pages outside of the valid\n",
      "range are returned with an empty list of items. To make the page sizes configurable,\n",
      "the value of the per_page argument is read from an application-specific configuration\n",
      "variable called FLASKY_POSTS_PER_PAGE that is added in config.py.\n",
      "With these changes, the blog post list on the home page will show a limited number\n",
      "of items. To see the second page of posts, add a ?page=2 query string to the URL in\n",
      "the browser’s address bar.\n",
      "Adding a Pagination Widget\n",
      "The return value of paginate() is an object of class Pagination, a class defined by\n",
      "Flask-SQLAlchemy. This object contains several properties that are useful to generate\n",
      "page links in a template, so it is passed to the template as an argument. A summary of\n",
      "the attributes of the pagination object is shown in Table 11-1.\n",
      "158 \n",
      "| \n",
      "Chapter 11: Blog Posts\n",
      "\n",
      "Table 11-1. Flask-SQLAlchemy pagination object attributes\n",
      "Attribute\n",
      "Description\n",
      "items\n",
      "The records in the current page\n",
      "query\n",
      "The source query that was paginated\n",
      "page\n",
      "The current page number\n",
      "prev_num\n",
      "The previous page number\n",
      "next_num\n",
      "The next page number\n",
      "has_next\n",
      "True if there is a next page\n",
      "has_prev\n",
      "True if there is a previous page\n",
      "pages\n",
      "The total number of pages for the query\n",
      "per_page The number of items per page\n",
      "total\n",
      "The total number of items returned by the query\n",
      "The pagination object also has some methods, listed in Table 11-2.\n",
      "Table 11-2. Flask-SQLAlchemy pagination object methods\n",
      "Method\n",
      "Description\n",
      "iter_pages(left_edge=2, \n",
      "left_current=2, \n",
      "right_current=5, \n",
      "right_edge=2)\n",
      "An iterator that returns the sequence of page numbers to display in a pagination\n",
      "widget. The list will have left_edge pages on the left side, left_current\n",
      "pages to the left of the current page, right_current pages to the right of the\n",
      "current page, and right_edge pages on the right side. For example, for page 50\n",
      "of 100 this iterator configured with default values will return the following pages: 1,\n",
      "2, None, 48, 49, 50, 51, 52, 53, 54, 55, None, 99, 100. A None value in the\n",
      "sequence indicates a gap in the sequence of pages.\n",
      "prev()\n",
      "A pagination object for the previous page.\n",
      "next()\n",
      "A pagination object for the next page.\n",
      "Armed with this powerful object and Bootstrap’s pagination CSS classes, it is quite\n",
      "easy to build a pagination footer in the template. The implementation shown in\n",
      "Example 11-10 is done as a reusable Jinja2 macro.\n",
      "Example 11-10. app/templates/_macros.html: pagination template macro\n",
      "{% macro pagination_widget(pagination, endpoint) %}\n",
      "<ul class=\"pagination\">\n",
      "    <li{% if not pagination.has_prev %} class=\"disabled\"{% endif %}>\n",
      "        <a href=\"{% if pagination.has_prev %}{{ url_for(endpoint,\n",
      "            page = pagination.page - 1, **kwargs) }}{% else %}#{% endif %}\">\n",
      "            &laquo;\n",
      "        </a>\n",
      "    </li>\n",
      "    {% for p in pagination.iter_pages() %}\n",
      "Paginating Long Blog Post Lists \n",
      "| \n",
      "159\n",
      "\n",
      "        {% if p %}\n",
      "            {% if p == pagination.page %}\n",
      "            <li class=\"active\">\n",
      "                <a href=\"{{ url_for(endpoint, page = p, **kwargs) }}\">{{ p }}</a>\n",
      "            </li>\n",
      "            {% else %}\n",
      "            <li>\n",
      "                <a href=\"{{ url_for(endpoint, page = p, **kwargs) }}\">{{ p }}</a>\n",
      "            </li>\n",
      "            {% endif %}\n",
      "        {% else %}\n",
      "        <li class=\"disabled\"><a href=\"#\">&hellip;</a></li>\n",
      "        {% endif %}\n",
      "    {% endfor %}\n",
      "    <li{% if not pagination.has_next %} class=\"disabled\"{% endif %}>\n",
      "        <a href=\"{% if pagination.has_next %}{{ url_for(endpoint,\n",
      "            page = pagination.page + 1, **kwargs) }}{% else %}#{% endif %}\">\n",
      "            &raquo;\n",
      "        </a>\n",
      "    </li>\n",
      "</ul>\n",
      "{% endmacro %}\n",
      "The macro creates a Bootstrap pagination element, which is a styled unordered list. It\n",
      "defines the following page links inside it:\n",
      "• A “previous page” link. This link gets the disabled CSS class if the current page\n",
      "is the first page.\n",
      "• Links to all pages returned by the pagination object’s iter_pages() iterator.\n",
      "These pages are rendered as links with an explicit page number, given as an argu‐\n",
      "ment to url_for(). The page currently displayed is highlighted using the active\n",
      "CSS class. Gaps in the sequence of pages are rendered with the ellipsis character.\n",
      "• A “next page” link. This link will appear disabled if the current page is the last\n",
      "page.\n",
      "Jinja2 macros always receive keyword arguments without having to include **kwargs\n",
      "in the argument list. The pagination macro passes all the keyword arguments it\n",
      "receives to the url_for() call that generates the pagination links. This approach can\n",
      "be used with routes such as the profile page that have a dynamic part.\n",
      "The pagination_widget macro can be added below the _posts.html template\n",
      "included by index.html and user.html. Example 11-11 shows how it is used in the\n",
      "application’s home page.\n",
      "160 \n",
      "| \n",
      "Chapter 11: Blog Posts\n",
      "\n",
      "Example 11-11. app/templates/index.html: pagination footer for blog post lists\n",
      "{% extends \"base.html\" %}\n",
      "{% import \"bootstrap/wtf.html\" as wtf %}\n",
      "{% import \"_macros.html\" as macros %}\n",
      "...\n",
      "{% include '_posts.html' %}\n",
      "<div class=\"pagination\">\n",
      "    {{ macros.pagination_widget(pagination, '.index') }}\n",
      "</div>\n",
      "{% endif %}\n",
      "Figure 11-2 shows how the pagination links appear in the page.\n",
      "Figure 11-2. Blog post pagination\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 11d to check out this version of the applica‐\n",
      "tion.\n",
      "Rich-Text Posts with Markdown and Flask-PageDown\n",
      "Plain-text posts are sufficient for short messages and status updates, but users who\n",
      "want to write longer articles will find the lack of formatting very limiting. In this sec‐\n",
      "tion, the text area field where posts are entered will be upgraded to support the Mark‐\n",
      "down syntax and present a rich-text preview of the post.\n",
      "Rich-Text Posts with Markdown and Flask-PageDown \n",
      "| \n",
      "161\n",
      "\n",
      "The implementation of this feature requires a few new packages:\n",
      "• PageDown, a client-side Markdown-to-HTML converter implemented in Java‐\n",
      "Script\n",
      "• Flask-PageDown, a PageDown wrapper for Flask that integrates PageDown with\n",
      "Flask-WTF forms\n",
      "• Markdown, a server-side Markdown-to-HTML converter implemented in\n",
      "Python\n",
      "• Bleach, an HTML sanitizer implemented in Python\n",
      "The Python packages can all be installed with pip:\n",
      "(venv) $ pip install flask-pagedown markdown bleach\n",
      "Using Flask-PageDown\n",
      "The Flask-PageDown extension defines a PageDownField class that has the same\n",
      "interface as the TextAreaField from WTForms. Before this field can be used, the\n",
      "extension needs to be initialized as shown in Example 11-12.\n",
      "Example 11-12. app/__init__.py: Flask-PageDown initialization\n",
      "from flask_pagedown import PageDown\n",
      "# ...\n",
      "pagedown = PageDown()\n",
      "# ...\n",
      "def create_app(config_name):\n",
      "    # ...\n",
      "    pagedown.init_app(app)\n",
      "    # ...\n",
      "To convert the text area control in the home page to a Markdown rich-text editor, the\n",
      "body field of the PostForm must be changed to a PageDownField as shown in\n",
      "Example 11-13.\n",
      "Example 11-13. app/main/forms.py: Markdown-enabled post form\n",
      "from flask_pagedown.fields import PageDownField\n",
      "class PostForm(FlaskForm):\n",
      "    body = PageDownField(\"What's on your mind?\", validators=[Required()])\n",
      "    submit = SubmitField('Submit')\n",
      "The Markdown preview is generated with the help of the PageDown libraries, so\n",
      "these must be added to the template. Flask-PageDown simplifies this task by provid‐\n",
      "162 \n",
      "| \n",
      "Chapter 11: Blog Posts\n",
      "\n",
      "ing a template macro that includes the required files from a CDN as shown in\n",
      "Example 11-14.\n",
      "Example 11-14. app/templates/index.html: Flask-PageDown template declaration\n",
      "{% block scripts %}\n",
      "{{ super() }}\n",
      "{{ pagedown.include_pagedown() }}\n",
      "{% endblock %}\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 11e to check out this version of the applica‐\n",
      "tion. To ensure that you have all the dependencies installed also run\n",
      "pip install -r requirements/dev.txt.\n",
      "With these changes, Markdown-formatted text typed in the text area field will be\n",
      "immediately rendered as HTML in the preview area below. Figure 11-3 shows the\n",
      "blog submission form with rich text.\n",
      "Figure 11-3. Rich-text blog post form\n",
      "Rich-Text Posts with Markdown and Flask-PageDown \n",
      "| \n",
      "163\n",
      "\n",
      "Handling Rich Text on the Server\n",
      "When the form is submitted, only the raw Markdown text is sent with the POST\n",
      "request; the HTML preview that is shown on the page is discarded. Sending the gen‐\n",
      "erated HTML preview with the form can be considered a security risk, as it would be\n",
      "fairly easy for an attacker to construct HTML sequences that do not match the Mark‐\n",
      "down source and submit them. To avoid any risks, only the Markdown source text is\n",
      "submitted, and once in the server it is converted again to HTML using Markdown, a\n",
      "Python Markdown-to-HTML converter. The resulting HTML is sanitized with\n",
      "Bleach to ensure that only a short list of allowed HTML tags are used.\n",
      "The conversion of the Markdown blog posts to HTML can be done in the _posts.html\n",
      "template, but this is inefficient as posts will have to be converted every time they are\n",
      "rendered to a page. To avoid this repetition, the conversion can be done once when\n",
      "the blog post is created and then cached in the database. The HTML code for the ren‐\n",
      "dered blog post is cached in a new field added to the Post model that the template\n",
      "can access directly. The original Markdown source is also kept in the database in case\n",
      "the post needs to be edited. Example 11-15 shows the changes to the Post model.\n",
      "Example 11-15. app/models.py: Markdown text handling in the Post model\n",
      "from markdown import markdown\n",
      "import bleach\n",
      "class Post(db.Model):\n",
      "    # ...\n",
      "    body_html = db.Column(db.Text)\n",
      "    # ...\n",
      "    @staticmethod\n",
      "    def on_changed_body(target, value, oldvalue, initiator):\n",
      "        allowed_tags = ['a', 'abbr', 'acronym', 'b', 'blockquote', 'code',\n",
      "                        'em', 'i', 'li', 'ol', 'pre', 'strong', 'ul',\n",
      "                        'h1', 'h2', 'h3', 'p']\n",
      "        target.body_html = bleach.linkify(bleach.clean(\n",
      "            markdown(value, output_format='html'),\n",
      "            tags=allowed_tags, strip=True))\n",
      "db.event.listen(Post.body, 'set', Post.on_changed_body)\n",
      "The on_changed_body() function is registered as a listener of SQLAlchemy’s “set”\n",
      "event for body, which means that it will be automatically invoked whenever the body\n",
      "field is set to a new value. The handler function renders the HTML version of the\n",
      "body and stores it in body_html, effectively making the conversion of the Markdown\n",
      "text to HTML fully automatic.\n",
      "The actual conversion is done in three steps. First, the markdown() function does an\n",
      "initial conversion to HTML. The result is passed to clean(), along with the list of\n",
      "164 \n",
      "| \n",
      "Chapter 11: Blog Posts\n",
      "\n",
      "approved HTML tags. The clean() function removes any tags not on the whitelist.\n",
      "The final conversion is done with linkify(), another function provided by Bleach\n",
      "that converts any URLs written in plain text into proper <a> links. This last step is\n",
      "necessary because automatic link generation is not officially in the Markdown specifi‐\n",
      "cation, but is a very convenient feature. On the client side, PageDown supports this\n",
      "feature as an optional extension, so linkify() matches that functionality on the\n",
      "server.\n",
      "The last change is to replace post.body with post.body_html in the template when\n",
      "available, as shown in Example 11-16.\n",
      "Example 11-16. app/templates/_posts.html: use the HTML version of the post bodies in\n",
      "the template\n",
      "...\n",
      "<div class=\"post-body\">\n",
      "    {% if post.body_html %}\n",
      "        {{ post.body_html | safe }}\n",
      "    {% else %}\n",
      "        {{ post.body }}\n",
      "    {% endif %}\n",
      "</div>\n",
      "...\n",
      "The | safe suffix when rendering the HTML body is there to tell Jinja2 not to escape\n",
      "the HTML elements. Jinja2 escapes all template variables by default as a security\n",
      "measure, but the Markdown-generated HTML was generated by the server, so it is\n",
      "safe to render directly as HTML.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 11f to check out this version of the applica‐\n",
      "tion. This update also contains a database migration, so remember\n",
      "to run flask db upgrade after you check out the code. To ensure\n",
      "that you have all the dependencies installed also run pip install\n",
      "-r requirements/dev.txt.\n",
      "Permanent Links to Blog Posts\n",
      "Users may want to share links to specific blog posts with friends on social networks.\n",
      "For this purpose, each post will be assigned a page with a unique URL that references\n",
      "it. The route and view function that support permanent links are shown in\n",
      "Example 11-17.\n",
      "Permanent Links to Blog Posts \n",
      "| \n",
      "165\n",
      "\n",
      "Example 11-17. app/main/views.py: enabling permanent links to posts\n",
      "@main.route('/post/<int:id>')\n",
      "def post(id):\n",
      "    post = Post.query.get_or_404(id)\n",
      "    return render_template('post.html', posts=[post])\n",
      "The URLs that will be assigned to blog posts are constructed with the unique id field\n",
      "assigned when the post is inserted in the database.\n",
      "For some types of applications, building permanent links that use\n",
      "readable URLs instead of numeric IDs may be preferred. An alter‐\n",
      "native to numeric IDs is to assign each blog post a slug, which is a\n",
      "unique string that is based on the title or first few words of the\n",
      "post.\n",
      "Note that the post.html template receives a list with a single element that is the post to\n",
      "render. Sending a list is a matter of convenience, so that the _posts.html template ref‐\n",
      "erenced by index.html and user.html can be used in this page as well.\n",
      "The permanent links are added at the bottom of each post in the generic _posts.html\n",
      "template, as shown in Example 11-18.\n",
      "Example 11-18. app/templates/_posts.html: adding permanent links to posts\n",
      "<ul class=\"posts\">\n",
      "    {% for post in posts %}\n",
      "    <li class=\"post\">\n",
      "        ...\n",
      "        <div class=\"post-content\">\n",
      "            ...\n",
      "            <div class=\"post-footer\">\n",
      "                <a href=\"{{ url_for('.post', id=post.id) }}\">\n",
      "                    <span class=\"label label-default\">Permalink</span>\n",
      "                </a>\n",
      "            </div>\n",
      "        </div>\n",
      "    </li>\n",
      "    {% endfor %}\n",
      "</ul>\n",
      "The new post.html template that renders the permanent link page is shown in\n",
      "Example 11-19. It includes the example template.\n",
      "166 \n",
      "| \n",
      "Chapter 11: Blog Posts\n",
      "\n",
      "Example 11-19. app/templates/post.html: permanent link template\n",
      "{% extends \"base.html\" %}\n",
      "{% block title %}Flasky - Post{% endblock %}\n",
      "{% block page_content %}\n",
      "{% include '_posts.html' %}\n",
      "{% endblock %}\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 11g to check out this version of the applica‐\n",
      "tion.\n",
      "Blog Post Editor\n",
      "The last feature related to blog posts is a post editor that allows users to edit their own\n",
      "posts. The blog post editor lives in a standalone page and is also based on Flask-\n",
      "PageDown, so a text area where the Markdown text of the blog post can be edited is\n",
      "followed by a rendered preview. The edit_post.html template is shown in\n",
      "Example 11-20.\n",
      "Example 11-20. app/templates/edit_post.html: edit blog post template\n",
      "{% extends \"base.html\" %}\n",
      "{% import \"bootstrap/wtf.html\" as wtf %}\n",
      "{% block title %}Flasky - Edit Post{% endblock %}\n",
      "{% block page_content %}\n",
      "<div class=\"page-header\">\n",
      "    <h1>Edit Post</h1>\n",
      "</div>\n",
      "<div>\n",
      "    {{ wtf.quick_form(form) }}\n",
      "</div>\n",
      "{% endblock %}\n",
      "{% block scripts %}\n",
      "{{ super() }}\n",
      "{{ pagedown.include_pagedown() }}\n",
      "{% endblock %}\n",
      "The route that supports the blog post editor is shown in Example 11-21.\n",
      "Blog Post Editor \n",
      "| \n",
      "167\n",
      "\n",
      "Example 11-21. app/main/views.py: edit blog post route\n",
      "@main.route('/edit/<int:id>', methods=['GET', 'POST'])\n",
      "@login_required\n",
      "def edit(id):\n",
      "    post = Post.query.get_or_404(id)\n",
      "    if current_user != post.author and \\\n",
      "            not current_user.can(Permission.ADMIN):\n",
      "        abort(403)\n",
      "    form = PostForm()\n",
      "    if form.validate_on_submit():\n",
      "        post.body = form.body.data\n",
      "        db.session.add(post)\n",
      "        db.session.commit()\n",
      "        flash('The post has been updated.')\n",
      "        return redirect(url_for('.post', id=post.id))\n",
      "    form.body.data = post.body\n",
      "    return render_template('edit_post.html', form=form)\n",
      "This view function is coded to allow only the author of a blog post to edit it, except\n",
      "for administrators, who are allowed to edit posts from all users. If a user tries to edit a\n",
      "post from another user, the view function responds with a 403 code. The PostForm\n",
      "web form class used here is the same one used on the home page.\n",
      "To complete the feature, a link to the blog post editor can be added below each blog\n",
      "post, next to the permanent link, as shown in Example 11-22.\n",
      "Example 11-22. app/templates/_posts.html: adding the edit blog post link\n",
      "<ul class=\"posts\">\n",
      "    {% for post in posts %}\n",
      "    <li class=\"post\">\n",
      "        ...\n",
      "        <div class=\"post-content\">\n",
      "            ...\n",
      "            <div class=\"post-footer\">\n",
      "                ...\n",
      "                {% if current_user == post.author %}\n",
      "                <a href=\"{{ url_for('.edit', id=post.id) }}\">\n",
      "                    <span class=\"label label-primary\">Edit</span>\n",
      "                </a>\n",
      "                {% elif current_user.is_administrator() %}\n",
      "                <a href=\"{{ url_for('.edit', id=post.id) }}\">\n",
      "                    <span class=\"label label-danger\">Edit [Admin]</span>\n",
      "                </a>\n",
      "                {% endif %}\n",
      "            </div>\n",
      "        </div>\n",
      "    </li>\n",
      "    {% endfor %}\n",
      "</ul>\n",
      "168 \n",
      "| \n",
      "Chapter 11: Blog Posts\n",
      "\n",
      "This change adds an “Edit” link to any blog posts that are authored by the current\n",
      "user. For administrators, the link is added to all posts. The administrator link is styled\n",
      "differently as a visual cue that this is an administration feature. Figure 11-4 shows\n",
      "how the Edit and Permalink links look in the web browser.\n",
      "Figure 11-4. Edit and Permalink links in a blog post\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 11h to check out this version of the applica‐\n",
      "tion.\n",
      "Blog Post Editor \n",
      "| \n",
      "169\n",
      "\n",
      "\n",
      "CHAPTER 12\n",
      "Followers\n",
      "Socially aware web applications allow users to connect with other users. Different\n",
      "applications call these relationships followers, friends, contacts, connections, or bud‐\n",
      "dies, but the feature is the same regardless of the name, and in all cases involves keep‐\n",
      "ing track of directional links between pairs of users and using these links in database\n",
      "queries.\n",
      "In this chapter, you will learn how to implement a follower feature for Flasky. Users\n",
      "will be able to “follow” other users and choose to filter the blog post list on the home\n",
      "page to include only those from the users they follow.\n",
      "Database Relationships Revisited\n",
      "As discussed in Chapter 5, databases establish links between records using relation‐\n",
      "ships. The one-to-many relationship is the most common type of relationship, where\n",
      "a record is linked with a list of related records. To implement this type of relationship,\n",
      "the elements on the “many” side have a foreign key that points to the linked element\n",
      "on the “one” side. The example application in its current state includes two one-to-\n",
      "many relationships: one that links user roles to lists of users and another that links\n",
      "users to the blog posts they authored.\n",
      "Most other relationship types can be derived from the one-to-many type. The many-\n",
      "to-one relationship is a one-to-many looked at from the point of view of the “many”\n",
      "side. The one-to-one relationship type is a simplification of the one-to-many, where\n",
      "the “many” side is constrained to have at most one element. The only relationship\n",
      "type that cannot be implemented as a simple variation of the one-to-many model is\n",
      "the many-to-many, which has lists of elements on both sides. This relationship is\n",
      "described in detail in the following section.\n",
      "171\n",
      "\n",
      "Many-to-Many Relationships\n",
      "The one-to-many, many-to-one, and one-to-one relationships all have at least one\n",
      "side with a single entity, so the links between related records are implemented with\n",
      "foreign keys pointing to that one element. But how do you implement a relationship\n",
      "where both sides are “many” sides?\n",
      "Consider the classic example of a many-to-many relationship: a database of students\n",
      "and the classes they are taking. Clearly, you can’t add a foreign key to a class in the\n",
      "students table, because a student takes many classes—one foreign key is not enough.\n",
      "Likewise, you cannot add a foreign key to a student in the classes table, because\n",
      "classes have more than one student. Both sides need a list of foreign keys.\n",
      "The solution is to add a third table to the database, called an association table. Now\n",
      "the many-to-many relationship can be decomposed into two one-to-many relation‐\n",
      "ships from each of the two original tables to the association table. Figure 12-1 shows\n",
      "how the many-to-many relationship between students and classes is represented.\n",
      "Figure 12-1. Many-to-many relationship example\n",
      "The association table in this example is called registrations. Each row in this table\n",
      "represents an individual registration of a student in a class.\n",
      "Querying a many-to-many relationship is a two-step process. To obtain the list of\n",
      "classes a student is taking, you start from the one-to-many relationship between stu‐\n",
      "dents and registrations and get the list of registrations for the desired student. Then\n",
      "the one-to-many relationship between classes and registrations is traversed in the\n",
      "many-to-one direction to obtain all the classes associated with the registrations\n",
      "retrieved for the student. Likewise, to find all the students in a class, you start from\n",
      "the class and get a list of registrations, then get the students linked to those registra‐\n",
      "tions.\n",
      "Traversing two relationships to obtain query results sounds difficult, but for a simple\n",
      "relationship like the one in the previous example, SQLAlchemy does most of the\n",
      "work. Following is the code that represents the many-to-many relationship in\n",
      "Figure 12-1:\n",
      "172 \n",
      "| \n",
      "Chapter 12: Followers\n",
      "\n",
      "registrations = db.Table('registrations',\n",
      "    db.Column('student_id', db.Integer, db.ForeignKey('students.id')),\n",
      "    db.Column('class_id', db.Integer, db.ForeignKey('classes.id'))\n",
      ")\n",
      "class Student(db.Model):\n",
      "    id = db.Column(db.Integer, primary_key=True)\n",
      "    name = db.Column(db.String)\n",
      "    classes = db.relationship('Class',\n",
      "                              secondary=registrations,\n",
      "                              backref=db.backref('students', lazy='dynamic'),\n",
      "                              lazy='dynamic')\n",
      "class Class(db.Model):\n",
      "    id = db.Column(db.Integer, primary_key=True)\n",
      "    name = db.Column(db.String)\n",
      "The relationship is defined with the same db.relationship() construct that is used\n",
      "for one-to-many relationships, but in the case of a many-to-many relationship the\n",
      "additional secondary argument must be set to the association table. The relationship\n",
      "can be defined in either one of the two classes, with the backref argument taking care\n",
      "of exposing the relationship from the other side as well. The association table is\n",
      "defined as a simple table, not as a model, since SQLAlchemy manages this table inter‐\n",
      "nally.\n",
      "The classes relationship uses list semantics, which makes working with a many-to-\n",
      "many relationship configured in this way extremely easy. Given a student s and a\n",
      "class c, the code that registers the student for the class is:\n",
      ">>> s.classes.append(c)\n",
      ">>> db.session.add(s)\n",
      "The queries that list the classes student s is registered for and the list of students reg‐\n",
      "istered for class c are also very simple:\n",
      ">>> s.classes.all()\n",
      ">>> c.students.all()\n",
      "The students relationship available in the Class model is the one defined in the\n",
      "db.backref() argument. Note that in this relationship the backref argument was\n",
      "expanded to also have a lazy='dynamic' attribute, so both sides return a query that\n",
      "can accept additional filters.\n",
      "If student s later decides to drop class c, you can update the database as follows:\n",
      ">>> s.classes.remove(c)\n",
      "Database Relationships Revisited \n",
      "| \n",
      "173\n",
      "\n",
      "Self-Referential Relationships\n",
      "A many-to-many relationship can be used to model users following other users, but\n",
      "there is a problem. In the example of students and classes, there were two very clearly\n",
      "defined entities linked together by the association table. However, to represent users\n",
      "following other users, it is just users—there is no second entity.\n",
      "A relationship in which both sides belong to the same table is said to be self-\n",
      "referential. In this case the entities on the left side of the relationship are users, which\n",
      "can be called the “followers.” The entities on the right side are also users, but these are\n",
      "the “followed” users. Conceptually, self-referential relationships are no different than\n",
      "regular relationships, but they are harder to think about. Figure 12-2 shows a data‐\n",
      "base diagram for a self-referential relationship that represents users following other\n",
      "users.\n",
      "Figure 12-2. Followers, many-to-many relationship\n",
      "The association table in this case is called follows. Each row in this table represents a\n",
      "user following another user. The one-to-many relationship pictured on the left side\n",
      "associates users with the list of “follows” rows in which they are the followers. The\n",
      "one-to-many relationship pictured on the right side associates users with the list of\n",
      "“follows” rows in which they are the followed user.\n",
      "Advanced Many-to-Many Relationships\n",
      "With a self-referential many-to-many relationship configured as shown in the previ‐\n",
      "ous example, the database can represent followers—but there is one limitation. A\n",
      "common need when working with many-to-many relationships is to store additional\n",
      "data that applies to the link between two entities. For the followers relationship, it can\n",
      "be useful to store the date a user started following another user, as that will enable\n",
      "lists of followers to be presented in chronological order. The only place this informa‐\n",
      "174 \n",
      "| \n",
      "Chapter 12: Followers\n",
      "\n",
      "tion can be stored is in the association table, but in an implementation similar to that\n",
      "of the students and classes shown earlier, the association table is an internal table that\n",
      "is fully managed by SQLAlchemy.\n",
      "To be able to work with custom data in the relationship, the association table must be\n",
      "promoted to a proper model that the application can access. Example 12-1 shows the\n",
      "new association table, represented by the Follow model.\n",
      "Example 12-1. app/models.py: the follows association table as a model\n",
      "class Follow(db.Model):\n",
      "    __tablename__ = 'follows'\n",
      "    follower_id = db.Column(db.Integer, db.ForeignKey('users.id'),\n",
      "                            primary_key=True)\n",
      "    followed_id = db.Column(db.Integer, db.ForeignKey('users.id'),\n",
      "                            primary_key=True)\n",
      "    timestamp = db.Column(db.DateTime, default=datetime.utcnow)\n",
      "SQLAlchemy cannot use the association table transparently because that will not give\n",
      "the application access to the custom fields in it. Instead, the many-to-many relation‐\n",
      "ship must be decomposed into the two basic one-to-many relationships for the left\n",
      "and right sides, and these must be defined as standard relationships. This is shown in\n",
      "Example 12-2.\n",
      "Example 12-2. app/models.py: a many-to-many relationship implemented as two one-\n",
      "to-many relationships\n",
      "class User(UserMixin, db.Model):\n",
      "    # ...\n",
      "    followed = db.relationship('Follow',\n",
      "                               foreign_keys=[Follow.follower_id],\n",
      "                               backref=db.backref('follower', lazy='joined'),\n",
      "                               lazy='dynamic',\n",
      "                               cascade='all, delete-orphan')\n",
      "    followers = db.relationship('Follow',\n",
      "                                foreign_keys=[Follow.followed_id],\n",
      "                                backref=db.backref('followed', lazy='joined'),\n",
      "                                lazy='dynamic',\n",
      "                                cascade='all, delete-orphan')\n",
      "Here the followed and followers relationships are defined as individual one-to-\n",
      "many relationships. Note that it is necessary to eliminate any ambiguity between for‐\n",
      "eign keys by specifying in each relationship which foreign key to use through the\n",
      "foreign_keys optional argument. The db.backref() arguments in these relation‐\n",
      "ships do not apply to each other; the back references are applied to the Follow model.\n",
      "Database Relationships Revisited \n",
      "| \n",
      "175\n",
      "\n",
      "The lazy argument for the back references is specified as joined. This lazy mode\n",
      "causes the related object to be loaded immediately from the join query. For example,\n",
      "if a user is following 100 other users, calling user.followed.all() will return a list\n",
      "of 100 Follow instances, where each one has the follower and followed back refer‐\n",
      "ence properties set to the respective users. The lazy='joined' mode enables this all\n",
      "to happen from a single database query. If lazy is set to the default value of select,\n",
      "then the follower and followed users are loaded lazily when they are first accessed\n",
      "and each attribute will require an individual query, which means that obtaining the\n",
      "complete list of followed users would require 100 additional database queries.\n",
      "The lazy argument on the User side of both relationships has different needs. These\n",
      "are on the “one” side and return the “many” side; here a mode of dynamic is used, so\n",
      "that the relationship attributes return query objects instead of returning the items\n",
      "directly. This allows additional filters to be added to the query before it is executed.\n",
      "The cascade argument configures how actions performed on a parent object propa‐\n",
      "gate to related objects. An example of a cascade option is the rule that says that when\n",
      "an object is added to the database session, any objects associated with it through rela‐\n",
      "tionships should automatically be added to the session as well. The default cascade\n",
      "options are appropriate for most situations, but there is one case in which the default\n",
      "cascade options do not work well for this many-to-many relationship. The default\n",
      "cascade behavior when an object is deleted is to set the foreign key in any related\n",
      "objects that link to it to a null value. But for an association table, the correct behavior\n",
      "is to delete the entries that point to a record that was deleted, as this effectively\n",
      "destroys the link. This is what the delete-orphan cascade option does.\n",
      "The value given to cascade is a comma-separated list of cascade\n",
      "options. This is somewhat confusing, but the option named all\n",
      "represents all the cascade options except delete-orphan. Using the\n",
      "value all, delete-orphan leaves the default cascade options\n",
      "enabled and adds the delete behavior for orphans.\n",
      "The application now needs to work with the two one-to-many relationships to imple‐\n",
      "ment the many-to-many functionality. Since these are operations that will need to be\n",
      "repeated often, it is a good idea to create helper methods in the User model for all the\n",
      "possible operations. The four new methods that control this relationship are shown in\n",
      "Example 12-3.\n",
      "176 \n",
      "| \n",
      "Chapter 12: Followers\n",
      "\n",
      "Example 12-3. app/models.py: followers helper methods\n",
      "class User(db.Model):\n",
      "    # ...\n",
      "    def follow(self, user):\n",
      "        if not self.is_following(user):\n",
      "            f = Follow(follower=self, followed=user)\n",
      "            db.session.add(f)\n",
      "    def unfollow(self, user):\n",
      "        f = self.followed.filter_by(followed_id=user.id).first()\n",
      "        if f:\n",
      "            db.session.delete(f)\n",
      "    def is_following(self, user):\n",
      "        if user.id is None:\n",
      "            return False\n",
      "        return self.followed.filter_by(\n",
      "            followed_id=user.id).first() is not None\n",
      "    def is_followed_by(self, user):\n",
      "        if user.id is None:\n",
      "            return False\n",
      "        return self.followers.filter_by(\n",
      "            follower_id=user.id).first() is not None\n",
      "The follow() method manually inserts a Follow instance in the association table that\n",
      "links a follower with a followed user, giving the application the opportunity to set the\n",
      "custom field. The two users who are connecting are manually assigned to the new\n",
      "Follow instance in its constructor, and then the object is added to the database ses‐\n",
      "sion as usual. Note that there is no need to manually set the timestamp field because\n",
      "it was defined with a default value that sets the current date and time. The\n",
      "unfollow() method uses the followed relationship to locate the Follow instance that\n",
      "links the user to the followed user who needs to be disconnected. To destroy the link\n",
      "between the two users, the Follow object is simply deleted. The is_following() and\n",
      "is_followed_by() methods search the left- and right-side one-to-many relation‐\n",
      "ships, respectively, for the given user and return True if the user is found. Both ensure\n",
      "that the given user has been assigned an id before issuing a query, to avoid errors if a\n",
      "user that has been created but not committed to the database yet is provided.\n",
      "Database Relationships Revisited \n",
      "| \n",
      "177\n",
      "\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 12a to check out this version of the applica‐\n",
      "tion. This update contains a database migration, so remember to\n",
      "run flask db upgrade after you check out the code.\n",
      "The database part of this feature is now complete. You can find a unit test that exerci‐\n",
      "ses the new database relationship in the source code repository on GitHub.\n",
      "Followers on the Profile Page\n",
      "The profile page of a user needs to present a “Follow” button if the user viewing it is\n",
      "not a follower, or an “Unfollow” button if the user is a follower. It is also a nice addi‐\n",
      "tion to show the follower and followed counts, display the lists of followers and fol‐\n",
      "lowed users, and show a “Follows you” sign when appropriate. The changes to the\n",
      "user profile template are shown in Example 12-4. Figure 12-3 shows how the addi‐\n",
      "tions look on the profile page.\n",
      "Example 12-4. app/templates/user.html: follower enhancements to the user profile\n",
      "header\n",
      "{% if current_user.can(Permission.FOLLOW) and user != current_user %}\n",
      "    {% if not current_user.is_following(user) %}\n",
      "    <a href=\"{{ url_for('.follow', username=user.username) }}\"\n",
      "        class=\"btn btn-primary\">Follow</a>\n",
      "    {% else %}\n",
      "    <a href=\"{{ url_for('.unfollow', username=user.username) }}\"\n",
      "        class=\"btn btn-default\">Unfollow</a>\n",
      "    {% endif %}\n",
      "{% endif %}\n",
      "<a href=\"{{ url_for('.followers', username=user.username) }}\">\n",
      "    Followers: <span class=\"badge\">{{ user.followers.count() }}</span>\n",
      "</a>\n",
      "<a href=\"{{ url_for('.followed_by', username=user.username) }}\">\n",
      "    Following: <span class=\"badge\">{{ user.followed.count() }}</span>\n",
      "</a>\n",
      "{% if current_user.is_authenticated and user != current_user and\n",
      "    user.is_following(current_user) %}\n",
      "| <span class=\"label label-default\">Follows you</span>\n",
      "{% endif %}\n",
      "178 \n",
      "| \n",
      "Chapter 12: Followers\n",
      "\n",
      "Figure 12-3. Followers on the profile page\n",
      "There are four new endpoints defined in these template changes. The /follow/<user‐\n",
      "name> route is invoked when a user clicks the “Follow” button on another user’s pro‐\n",
      "file page. The implementation is shown in Example 12-5.\n",
      "Example 12-5. app/main/views.py: follow route and view function\n",
      "@main.route('/follow/<username>')\n",
      "@login_required\n",
      "@permission_required(Permission.FOLLOW)\n",
      "def follow(username):\n",
      "    user = User.query.filter_by(username=username).first()\n",
      "    if user is None:\n",
      "        flash('Invalid user.')\n",
      "        return redirect(url_for('.index'))\n",
      "    if current_user.is_following(user):\n",
      "        flash('You are already following this user.')\n",
      "        return redirect(url_for('.user', username=username))\n",
      "    current_user.follow(user)\n",
      "    db.session.commit()\n",
      "    flash('You are now following %s.' % username)\n",
      "    return redirect(url_for('.user', username=username))\n",
      "This view function loads the requested user, verifies that it is valid and that it isn’t\n",
      "already followed by the logged-in user, and then calls the follow() helper function in\n",
      "Followers on the Profile Page \n",
      "| \n",
      "179\n",
      "\n",
      "the User model to establish the link. The /unfollow/<username> route is implemented\n",
      "in a similar way.\n",
      "The /followers/<username> route is invoked when a user clicks another user’s fol‐\n",
      "lower count on the profile page. The implementation is shown in Example 12-6.\n",
      "Example 12-6. app/main/views.py: followers route and view function\n",
      "@main.route('/followers/<username>')\n",
      "def followers(username):\n",
      "    user = User.query.filter_by(username=username).first()\n",
      "    if user is None:\n",
      "        flash('Invalid user.')\n",
      "        return redirect(url_for('.index'))\n",
      "    page = request.args.get('page', 1, type=int)\n",
      "    pagination = user.followers.paginate(\n",
      "        page, per_page=current_app.config['FLASKY_FOLLOWERS_PER_PAGE'],\n",
      "        error_out=False)\n",
      "    follows = [{'user': item.follower, 'timestamp': item.timestamp}\n",
      "               for item in pagination.items]\n",
      "    return render_template('followers.html', user=user, title=\"Followers of\",\n",
      "                           endpoint='.followers', pagination=pagination,\n",
      "                           follows=follows)\n",
      "This function loads and validates the requested user, then paginates its followers\n",
      "relationship using the same techniques learned in Chapter 11. Because the query for\n",
      "followers returns Follow instances, the list is converted into another list that has user\n",
      "and timestamp fields in each entry so that rendering is simpler.\n",
      "The template that renders the follower list can be written generically so that it can be\n",
      "used for lists of followers and followed users. The template receives the user, a title for\n",
      "the page, the endpoint to use in the pagination links, the pagination object, and the\n",
      "list of results.\n",
      "The followed_by endpoint is almost identical. The only difference is that the list of\n",
      "users is obtained from the user.followed relationship. The template arguments are\n",
      "also adjusted accordingly.\n",
      "The followers.html template is implemented with a two-column table that shows user‐\n",
      "names and their avatars on the left and Flask-Moment timestamps on the right. You\n",
      "can consult the source code repository on GitHub to study the implementation in\n",
      "detail.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 12b to check out this version of the applica‐\n",
      "tion.\n",
      "180 \n",
      "| \n",
      "Chapter 12: Followers\n",
      "\n",
      "Querying Followed Posts Using a Database Join\n",
      "The application’s home page currently shows all the posts in the database in descend‐\n",
      "ing chronological order. With the followers feature now complete, it would be a nice\n",
      "addition to give users the option to view blog posts from only the users they follow.\n",
      "The obvious way to load all the posts authored by followed users is to first get the list\n",
      "of those users and then get the posts from each and sort them into a single list. Of\n",
      "course, that approach does not scale well; the effort required to obtain this combined\n",
      "list will grow as the database grows, and operations such as pagination cannot be\n",
      "done efficiently. This problem is commonly known as the “N+1 problem,” because\n",
      "working with the database in this way requires issuing N+1 database queries, with N\n",
      "being the number of results returned by the first query. The key to obtaining the blog\n",
      "posts with good performance regardless of the database size is doing it all with a sin‐\n",
      "gle query.\n",
      "The database operation that can do this is called a join. A join operation takes two or\n",
      "more tables and finds all the combinations of rows that satisfy a given condition. The\n",
      "resulting combined rows are inserted into a temporary table that is the result of the\n",
      "join. The best way to explain how joins work is through an example.\n",
      "Table 12-1 shows an example users table with three users.\n",
      "Table 12-1. users table\n",
      "id username\n",
      "1\n",
      "john\n",
      "2\n",
      "susan\n",
      "3\n",
      "david\n",
      "Table 12-2 shows the corresponding posts table, with some blog posts.\n",
      "Table 12-2. posts table\n",
      "id author_id body\n",
      "1\n",
      "2\n",
      "Blog post by susan\n",
      "2\n",
      "1\n",
      "Blog post by john\n",
      "3\n",
      "3\n",
      "Blog post by david\n",
      "4\n",
      "1\n",
      "Second blog post by john\n",
      "Finally, Table 12-3 shows who is following whom. In this table you can see that john is\n",
      "following david, susan is following john and david, and david is not following anyone.\n",
      "Querying Followed Posts Using a Database Join \n",
      "| \n",
      "181\n",
      "\n",
      "Table 12-3. follows table\n",
      "follower_id\n",
      "followed_id\n",
      "1\n",
      "3\n",
      "2\n",
      "1\n",
      "2\n",
      "3\n",
      "To obtain the list of posts by users followed by the user susan, the posts and follows\n",
      "tables must be combined. First the follows table is filtered to keep just the rows that\n",
      "have susan as the follower, which in this example are the last two rows. Then a tem‐\n",
      "porary join table is created from all the possible combinations of rows from the posts\n",
      "and filtered follows tables in which the author_id of the post is the same as the\n",
      "followed_id of the follow, effectively selecting any posts that appear in the list of\n",
      "users susan is following. Table 12-4 shows the result of the join operation. The col‐\n",
      "umns that were used to perform the join are marked with an * in this table.\n",
      "Table 12-4. Joined table\n",
      "id author_id*\n",
      "body\n",
      "follower_id\n",
      "followed_id*\n",
      "2\n",
      "1\n",
      "Blog post by john\n",
      "2\n",
      "1\n",
      "3\n",
      "3\n",
      "Blog post by david\n",
      "2\n",
      "3\n",
      "4\n",
      "1\n",
      "Second blog post by john\n",
      "2\n",
      "1\n",
      "This table contains exactly the list of blog posts authored by users that susan is follow‐\n",
      "ing. The Flask-SQLAlchemy query that performs the join operation as described is\n",
      "fairly complex:\n",
      "return db.session.query(Post).select_from(Follow).\\\n",
      "    filter_by(follower_id=self.id).\\\n",
      "    join(Post, Follow.followed_id == Post.author_id)\n",
      "All the queries that you have seen so far start from the query attribute of the model\n",
      "that is queried. That format does not work well for this query, because the query\n",
      "needs to return posts rows, yet the first operation that needs to be done is to apply a\n",
      "filter to the follows table. So, a more basic form of the query is used instead. To fully\n",
      "understand this query, each part should be looked at individually:\n",
      "• db.session.query(Post) specifies that this is going to be a query that returns\n",
      "Post objects.\n",
      "• select_from(Follow) says that the query begins with the Follow model.\n",
      "• filter_by(follower_id=self.id) performs the filtering of the follows table by\n",
      "the follower user.\n",
      "182 \n",
      "| \n",
      "Chapter 12: Followers\n",
      "\n",
      "• join(Post, Follow.followed_id == Post.author_id) joins the results of\n",
      "filter_by() with the Post objects.\n",
      "The query can be simplified by swapping the order of the filter and the join:\n",
      "return Post.query.join(Follow, Follow.followed_id == Post.author_id)\\\n",
      "    .filter(Follow.follower_id == self.id)\n",
      "Issuing the join operation first means the query can be started from Post.query, so\n",
      "now the only two filters that need to be applied are join() and filter(). It may\n",
      "seem that doing the join first and then the filtering would be more work, but in real‐\n",
      "ity these two queries are equivalent. SQLAlchemy first collects all the filters and then\n",
      "generates the query in the most efficient way. The native SQL instructions for these\n",
      "two queries are nearly identical, something that you can confirm by printing the\n",
      "query object converted to a string (i.e., print(str(query))). The final version of this\n",
      "query is added to the Post model, as shown in Example 12-7.\n",
      "Example 12-7. app/models.py: obtaining followed posts\n",
      "class User(db.Model):\n",
      "    # ...\n",
      "    @property\n",
      "    def followed_posts(self):\n",
      "        return Post.query.join(Follow, Follow.followed_id == Post.author_id)\\\n",
      "            .filter(Follow.follower_id == self.id)\n",
      "Note that the followed_posts() method is defined as a property so that it does not\n",
      "need the (). That way, all relationships have a consistent syntax.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 12c to check out this version of the applica‐\n",
      "tion.\n",
      "Joins are extremely hard to wrap your head around; you may need to experiment\n",
      "with the example code in a shell before it all sinks in.\n",
      "Showing Followed Posts on the Home Page\n",
      "The home page can now give users the choice to view all blog posts or just those from\n",
      "followed users. Example 12-8 shows how this choice is implemented.\n",
      "Showing Followed Posts on the Home Page \n",
      "| \n",
      "183\n",
      "\n",
      "Example 12-8. app/main/views.py: showing all or followed posts\n",
      "@main.route('/', methods = ['GET', 'POST'])\n",
      "def index():\n",
      "    # ...\n",
      "    show_followed = False\n",
      "    if current_user.is_authenticated:\n",
      "        show_followed = bool(request.cookies.get('show_followed', ''))\n",
      "    if show_followed:\n",
      "        query = current_user.followed_posts\n",
      "    else:\n",
      "        query = Post.query\n",
      "    pagination = query.order_by(Post.timestamp.desc()).paginate(\n",
      "        page, per_page=current_app.config['FLASKY_POSTS_PER_PAGE'],\n",
      "        error_out=False)\n",
      "    posts = pagination.items\n",
      "    return render_template('index.html', form=form, posts=posts,\n",
      "                           show_followed=show_followed, pagination=pagination)\n",
      "The choice of showing all or followed posts is stored in a cookie called\n",
      "show_followed that, when set to a nonempty string, indicates that only followed\n",
      "posts should be shown. Cookies are stored in the request object as a\n",
      "request.cookies dictionary. The string value of the cookie is converted to a Boolean,\n",
      "and based on its value a query local variable is set to the query that obtains the com‐\n",
      "plete or filtered list of blog posts. To show all the posts, the top-level query\n",
      "Post.query is used, and the recently added User.followed_posts property is used\n",
      "when the list should be restricted to followed users. The query stored in the query\n",
      "local variable is then paginated and the results sent to the template as before.\n",
      "The show_followed cookie is set in two new routes, shown in Example 12-9.\n",
      "Example 12-9. app/main/views.py: selection of all or followed posts\n",
      "@main.route('/all')\n",
      "@login_required\n",
      "def show_all():\n",
      "    resp = make_response(redirect(url_for('.index')))\n",
      "    resp.set_cookie('show_followed', '', max_age=30*24*60*60)  # 30 days\n",
      "    return resp\n",
      "@main.route('/followed')\n",
      "@login_required\n",
      "def show_followed():\n",
      "    resp = make_response(redirect(url_for('.index')))\n",
      "    resp.set_cookie('show_followed', '1', max_age=30*24*60*60)  # 30 days\n",
      "    return resp\n",
      "184 \n",
      "| \n",
      "Chapter 12: Followers\n",
      "\n",
      "Links to these routes are added to the home page template. When they are invoked,\n",
      "the show_followed cookie is set to the proper value and a redirect back to the home\n",
      "page is issued.\n",
      "Cookies can be set only on a response object, so these routes need to create a\n",
      "response object through make_response() instead of letting Flask do this.\n",
      "The set_cookie() function takes the cookie name and the value as the first two argu‐\n",
      "ments. The max_age optional argument sets the number of seconds until the cookie\n",
      "expires. Not including this argument makes the cookie expire when the browser win‐\n",
      "dow is closed. In this case, a maximum age of 30 days is set so that the setting is\n",
      "remembered even if the user does not return to the application for several days.\n",
      "The changes to the template add two navigation tabs at the top of the page that invoke\n",
      "the /all or /followed routes to set the correct settings in the session. You can inspect\n",
      "the template changes in detail in the source code repository on GitHub. Figure 12-4\n",
      "shows how the home page looks with these changes.\n",
      "Figure 12-4. Followed posts on the home page\n",
      "Showing Followed Posts on the Home Page \n",
      "| \n",
      "185\n",
      "\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 12d to check out this version of the applica‐\n",
      "tion.\n",
      "If you try the application at this point and switch to the followed list of posts, you will\n",
      "notice that your own posts do not appear in the list. This is of course correct, because\n",
      "users are not followers of themselves.\n",
      "Even though the queries are working as designed, most users will expect to see their\n",
      "own posts when they are looking at those of their friends. The easiest way to address\n",
      "this issue is to register all users as their own followers at the time they are created.\n",
      "This trick is shown in Example 12-10.\n",
      "Example 12-10. app/models.py: making users their own followers when they are created\n",
      "class User(UserMixin, db.Model):\n",
      "    # ...\n",
      "    def __init__(self, **kwargs):\n",
      "        # ...\n",
      "        self.follow(self)\n",
      "Unfortunately, you likely have several users in the database who are already created\n",
      "and are not following themselves. If the database is small and easy to regenerate, then\n",
      "it can be deleted and re-created, but if that is not an option, then adding an update\n",
      "function that fixes existing users is the proper solution. This is shown in\n",
      "Example 12-11.\n",
      "Example 12-11. app/models.py: making users their own followers\n",
      "class User(UserMixin, db.Model):\n",
      "    # ...\n",
      "    @staticmethod\n",
      "    def add_self_follows():\n",
      "        for user in User.query.all():\n",
      "            if not user.is_following(user):\n",
      "                user.follow(user)\n",
      "                db.session.add(user)\n",
      "                db.session.commit()\n",
      "    # ...\n",
      "Now the database can be updated by running the previous example function from the\n",
      "shell:\n",
      "(venv) $ flask shell\n",
      ">>> User.add_self_follows()\n",
      "186 \n",
      "| \n",
      "Chapter 12: Followers\n",
      "\n",
      "Creating functions that introduce updates to the database is a common technique\n",
      "used to update applications that are deployed, as running a scripted update is less\n",
      "error prone than updating databases manually. In Chapter 17 you will see how this\n",
      "function and others like it can be incorporated into a deployment script.\n",
      "Making all users self-followers makes the application more usable, but this change\n",
      "introduces a few complications. The follower and followed user counts shown in the\n",
      "user profile page are now increased by one due to the self-follower links. The num‐\n",
      "bers need to be decreased by one to be accurate, which is easy to do directly in the\n",
      "template \n",
      "by \n",
      "rendering \n",
      "{{ \n",
      "user.followers.count() \n",
      "- \n",
      "1 \n",
      "}} \n",
      "and\n",
      "{{ user.followed.count() - 1 }}. The lists of followers and followed users also\n",
      "must be adjusted to not show the same user, another simple task to do in the template\n",
      "with a conditional. Finally, any unit tests that check follower counts are also affected\n",
      "by the self-follower links and must be adjusted to account for the self-followers.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 12e to check out this version of the applica‐\n",
      "tion.\n",
      "In the next chapter, the user comment subsystem will be implemented—another very\n",
      "important feature of socially aware applications.\n",
      "Showing Followed Posts on the Home Page \n",
      "| \n",
      "187\n",
      "\n",
      "\n",
      "CHAPTER 13\n",
      "User Comments\n",
      "Allowing users to interact is key to the success of a social blogging platform. In this\n",
      "chapter, you will learn how to implement user comments. The techniques presented\n",
      "are generic enough to be directly applicable to a large number of socially enabled\n",
      "applications.\n",
      "Database Representation of Comments\n",
      "Comments are not very different from blog posts. Both have a body, an author, and a\n",
      "timestamp, and in this particular implementation both are written with Markdown\n",
      "syntax. Figure 13-1 shows a diagram of the comments table and its relationships with\n",
      "other tables in the database.\n",
      "Figure 13-1. Database representation of blog post comments\n",
      "Comments apply to specific blog posts, so a one-to-many relationship from the posts\n",
      "table is defined. This relationship can be used to obtain the list of comments associ‐\n",
      "ated with a particular blog post.\n",
      "The comments table is also in a one-to-many relationship with the users table. This\n",
      "relationship gives access to all the comments made by a user, and indirectly how\n",
      "189\n",
      "\n",
      "many comments a user has written, a piece of information that can be interesting to\n",
      "show in user profile pages. The definition of the Comment model is shown in\n",
      "Example 13-1.\n",
      "Example 13-1. app/models.py: comment model\n",
      "class Comment(db.Model):\n",
      "    __tablename__ = 'comments'\n",
      "    id = db.Column(db.Integer, primary_key=True)\n",
      "    body = db.Column(db.Text)\n",
      "    body_html = db.Column(db.Text)\n",
      "    timestamp = db.Column(db.DateTime, index=True, default=datetime.utcnow)\n",
      "    disabled = db.Column(db.Boolean)\n",
      "    author_id = db.Column(db.Integer, db.ForeignKey('users.id'))\n",
      "    post_id = db.Column(db.Integer, db.ForeignKey('posts.id'))\n",
      "    @staticmethod\n",
      "    def on_changed_body(target, value, oldvalue, initiator):\n",
      "        allowed_tags = ['a', 'abbr', 'acronym', 'b', 'code', 'em', 'i',\n",
      "                        'strong']\n",
      "        target.body_html = bleach.linkify(bleach.clean(\n",
      "            markdown(value, output_format='html'),\n",
      "            tags=allowed_tags, strip=True))\n",
      "db.event.listen(Comment.body, 'set', Comment.on_changed_body)\n",
      "The attributes of the Comment model are almost the same as those of Post. One addi‐\n",
      "tion is the disabled field, a Boolean that will be used by moderators to suppress com‐\n",
      "ments that are inappropriate or offensive. Like blog posts, comments define an event\n",
      "that triggers any time the body field changes, automating the rendering of the Mark‐\n",
      "down text to HTML. The process is identical to what was done for blog posts in\n",
      "Chapter 11, but since comments tend to be short, the list of HTML tags that are\n",
      "allowed in the conversion from Markdown is more restrictive, the paragraph-related\n",
      "tags have been removed, and only the character formatting tags are left.\n",
      "To complete the database changes, the User and Post models must define the one-to-\n",
      "many relationships with the comments table, as shown in Example 13-2.\n",
      "Example 13-2. app/models.py: one-to-many relationships from users and posts to\n",
      "comments\n",
      "class User(db.Model):\n",
      "    # ...\n",
      "    comments = db.relationship('Comment', backref='author', lazy='dynamic')\n",
      "class Post(db.Model):\n",
      "    # ...\n",
      "    comments = db.relationship('Comment', backref='post', lazy='dynamic')\n",
      "190 \n",
      "| \n",
      "Chapter 13: User Comments\n",
      "\n",
      "Comment Submission and Display\n",
      "In this application, comments are displayed on the individual blog post pages that\n",
      "were added as permanent links in Chapter 11. A submission form is also included on\n",
      "these pages. Example 13-3 shows the web form that will be used to enter comments—\n",
      "an extremely simple form that only has a text field and a submit button.\n",
      "Example 13-3. app/main/forms.py: comment input form\n",
      "class CommentForm(FlaskForm):\n",
      "    body = StringField('', validators=[DataRequired()])\n",
      "    submit = SubmitField('Submit')\n",
      "Example 13-4 shows the updated /post/<int:id> route with support for comments.\n",
      "Example 13-4. app/main/views.py: blog post comments support\n",
      "@main.route('/post/<int:id>', methods=['GET', 'POST'])\n",
      "def post(id):\n",
      "    post = Post.query.get_or_404(id)\n",
      "    form = CommentForm()\n",
      "    if form.validate_on_submit():\n",
      "        comment = Comment(body=form.body.data,\n",
      "                          post=post,\n",
      "                          author=current_user._get_current_object())\n",
      "        db.session.add(comment)\n",
      "        db.session.commit()\n",
      "        flash('Your comment has been published.')\n",
      "        return redirect(url_for('.post', id=post.id, page=-1))\n",
      "    page = request.args.get('page', 1, type=int)\n",
      "    if page == -1:\n",
      "        page = (post.comments.count() - 1) // \\\n",
      "               current_app.config['FLASKY_COMMENTS_PER_PAGE'] + 1\n",
      "    pagination = post.comments.order_by(Comment.timestamp.asc()).paginate(\n",
      "        page, per_page=current_app.config['FLASKY_COMMENTS_PER_PAGE'],\n",
      "        error_out=False)\n",
      "    comments = pagination.items\n",
      "    return render_template('post.html', posts=[post], form=form,\n",
      "                           comments=comments, pagination=pagination)\n",
      "This view function instantiates the comment form and sends it to the post.html tem‐\n",
      "plate for rendering. The logic that inserts a new comment when the form is submitted\n",
      "is similar to the handling of blog posts. As in the Post case, the author of the com‐\n",
      "ment cannot be set directly to current_user because this is a context variable proxy\n",
      "object. The expression current_user._get_current_object() returns the actual\n",
      "User object.\n",
      "Comment Submission and Display \n",
      "| \n",
      "191\n",
      "\n",
      "The comments are sorted by their timestamp in chronological order, so new com‐\n",
      "ments are always added at the bottom of the list. When a new comment is entered,\n",
      "the redirect that ends the request goes back to the same URL, but the url_for()\n",
      "function sets the page to -1, a special page number that is used to request the last\n",
      "page of comments so that the comment just entered is seen on the page. When the\n",
      "page number is obtained from the query string and found to be -1, a calculation with\n",
      "the number of comments and the page size is done to obtain the actual page number\n",
      "to use.\n",
      "The list of comments associated with the post is obtained through the post.comments\n",
      "one-to-many relationship, sorted by comment timestamp, and paginated with the\n",
      "same techniques used for blog posts. The comments and the pagination object are\n",
      "sent to the template for rendering. The FLASKY_COMMENTS_PER_PAGE configuration\n",
      "variable is added to config.py to control the size of each page of comments.\n",
      "The comment rendering is defined in a new template, _comments.html, that is similar\n",
      "to _posts.html but uses a different set of CSS classes. This template is included by\n",
      "_posts.html below the body of the post, followed by a call to the pagination macro.\n",
      "You can review the changes to the templates in the application’s GitHub repository.\n",
      "To complete this feature, blog posts shown on the home and profile pages need links\n",
      "to the pages with the comments. This is shown in Example 13-5.\n",
      "Example 13-5. _app/templates/_posts.html: link to blog post comments\n",
      "<a href=\"{{ url_for('.post', id=post.id) }}#comments\">\n",
      "    <span class=\"label label-primary\">\n",
      "        {{ post.comments.count() }} Comments\n",
      "    </span>\n",
      "</a>\n",
      "Note how the text of the link includes the number of comments, which is easily\n",
      "obtained from the one-to-many relationship between the posts and comments tables\n",
      "using SQLAlchemy’s count() filter.\n",
      "Also of interest is the structure of the link to the comments page, which is built as the\n",
      "permanent link for the post with a #comments suffix added. This last part is called a\n",
      "URL fragment and is used to indicate an initial scroll position for the page. The web\n",
      "browser looks for an element with the id given and scrolls the page so that element\n",
      "appears at the top of the page. This initial position is set to the “Comments” heading\n",
      "in the post.html template, which is written as <h4 id=\"comments\">Comments</h4>.\n",
      "Figure 13-2 shows how the comments appear on the page.\n",
      "192 \n",
      "| \n",
      "Chapter 13: User Comments\n",
      "\n",
      "Figure 13-2. Blog post comments\n",
      "An additional change was made to the pagination macro. The pagination links for\n",
      "comments also need the #comments fragment added, so a fragment argument was\n",
      "added to the macro and passed in the macro invocation from the post.html template.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 13a to check out this version of the applica‐\n",
      "tion. This update contains a database migration, so remember to\n",
      "run flask db upgrade after you check out the code.\n",
      "Comment Moderation\n",
      "In Chapter 9 a list of user roles was defined, each with a list of permissions. One of\n",
      "the permissions was Permission.MODERATE, which gives users who have it in their\n",
      "roles the power to moderate comments made by others.\n",
      "This feature will be exposed as a link in the navigation bar that appears only to users\n",
      "who are permitted to use it. This is done in the base.html template using a condi‐\n",
      "tional, as shown in Example 13-6.\n",
      "Comment Moderation \n",
      "| \n",
      "193\n",
      "\n",
      "Example 13-6. app/templates/base.html: Moderate Comments link in navigation bar\n",
      "...\n",
      "{% if current_user.can(Permission.MODERATE) %}\n",
      "<li><a href=\"{{ url_for('main.moderate') }}\">Moderate Comments</a></li>\n",
      "{% endif %}\n",
      "...\n",
      "The moderation page shows the comments for all the posts in the same list, with the\n",
      "most recent comments shown first. Below each comment is a button that can toggle\n",
      "the disabled attribute. The /moderate route is shown in Example 13-7.\n",
      "Example 13-7. app/main/views.py: comment moderation route\n",
      "@main.route('/moderate')\n",
      "@login_required\n",
      "@permission_required(Permission.MODERATE)\n",
      "def moderate():\n",
      "    page = request.args.get('page', 1, type=int)\n",
      "    pagination = Comment.query.order_by(Comment.timestamp.desc()).paginate(\n",
      "        page, per_page=current_app.config['FLASKY_COMMENTS_PER_PAGE'],\n",
      "        error_out=False)\n",
      "    comments = pagination.items\n",
      "    return render_template('moderate.html', comments=comments,\n",
      "                           pagination=pagination, page=page)\n",
      "This is a very simple function that reads a page of comments from the database and\n",
      "passes them on to a template for rendering. Along with the comments, the template\n",
      "receives the pagination object and the current page number.\n",
      "The moderate.html template, shown in Example 13-8, is also simple because it relies\n",
      "on the _comments.html subtemplate created earlier for the rendering of the com‐\n",
      "ments.\n",
      "Example 13-8. app/templates/moderate.html: comment moderation template\n",
      "{% extends \"base.html\" %}\n",
      "{% import \"_macros.html\" as macros %}\n",
      "{% block title %}Flasky - Comment Moderation{% endblock %}\n",
      "{% block page_content %}\n",
      "<div class=\"page-header\">\n",
      "    <h1>Comment Moderation</h1>\n",
      "</div>\n",
      "{% set moderate = True %}\n",
      "{% include '_comments.html' %}\n",
      "{% if pagination %}\n",
      "<div class=\"pagination\">\n",
      "194 \n",
      "| \n",
      "Chapter 13: User Comments\n",
      "\n",
      "    {{ macros.pagination_widget(pagination, '.moderate') }}\n",
      "</div>\n",
      "{% endif %}\n",
      "{% endblock %}\n",
      "This template defers the rendering of the comments to the _comments.html template,\n",
      "but before it hands control to the subordinate template it uses Jinja2’s set directive to\n",
      "define a moderate template variable set to True. This variable is used by the _com‐\n",
      "ments.html template to determine whether the moderation features need to be ren‐\n",
      "dered.\n",
      "The portion of the _comments.html template that renders the body of each comment\n",
      "needs to be modified in two ways. For regular users (when the moderate variable is\n",
      "not set), any comments that are marked as disabled should be suppressed. For mod‐\n",
      "erators (when moderate is set to True), the body of the comment must be rendered\n",
      "regardless of the disabled state, and below the body a button should be included to\n",
      "toggle the state. Example 13-9 shows these changes.\n",
      "Example 13-9. app/templates/_comments.html: rendering of the comment bodies\n",
      "...\n",
      "<div class=\"comment-body\">\n",
      "    {% if comment.disabled %}\n",
      "    <p></p><i>This comment has been disabled by a moderator.</i></p>\n",
      "    {% endif %}\n",
      "    {% if moderate or not comment.disabled %}\n",
      "        {% if comment.body_html %}\n",
      "            {{ comment.body_html | safe }}\n",
      "        {% else %}\n",
      "            {{ comment.body }}\n",
      "        {% endif %}\n",
      "    {% endif %}\n",
      "</div>\n",
      "{% if moderate %}\n",
      "    <br>\n",
      "    {% if comment.disabled %}\n",
      "    <a class=\"btn btn-default btn-xs\" href=\"{{ url_for('.moderate_enable',\n",
      "        id=comment.id, page=page) }}\">Enable</a>\n",
      "    {% else %}\n",
      "    <a class=\"btn btn-danger btn-xs\" href=\"{{ url_for('.moderate_disable',\n",
      "        id=comment.id, page=page) }}\">Disable</a>\n",
      "    {% endif %}\n",
      "{% endif %}\n",
      "...\n",
      "With these changes, users will see a short notice for disabled comments. Moderators\n",
      "will see both the notice and the comment body. Moderators will also see a button to\n",
      "toggle the disabled state below each comment. The button invokes one of two new\n",
      "Comment Moderation \n",
      "| \n",
      "195\n",
      "\n",
      "routes, depending on which of the two possible states the comment is changing to.\n",
      "Example 13-10 shows how these routes are defined.\n",
      "Example 13-10. app/main/views.py: comment moderation routes\n",
      "@main.route('/moderate/enable/<int:id>')\n",
      "@login_required\n",
      "@permission_required(Permission.MODERATE)\n",
      "def moderate_enable(id):\n",
      "    comment = Comment.query.get_or_404(id)\n",
      "    comment.disabled = False\n",
      "    db.session.add(comment)\n",
      "    return redirect(url_for('.moderate',\n",
      "                            page=request.args.get('page', 1, type=int)))\n",
      "@main.route('/moderate/disable/<int:id>')\n",
      "@login_required\n",
      "@permission_required(Permission.MODERATE)\n",
      "def moderate_disable(id):\n",
      "    comment = Comment.query.get_or_404(id)\n",
      "    comment.disabled = True\n",
      "    db.session.add(comment)\n",
      "    return redirect(url_for('.moderate',\n",
      "                            page=request.args.get('page', 1, type=int)))\n",
      "The comment enable and disable routes load the comment object, set the disabled\n",
      "field to the proper value, and write it back to the database. At the end, they redirect\n",
      "back to the comment moderation page (shown in Figure 13-3), and if a page argu‐\n",
      "ment was given in the query string, they include it in the redirect. The buttons in the\n",
      "_comments.html template are rendered with the page argument so that the redirect\n",
      "brings the user back to the same page.\n",
      "196 \n",
      "| \n",
      "Chapter 13: User Comments\n",
      "\n",
      "Figure 13-3. Comment moderation page\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 13b to check out this version of the applica‐\n",
      "tion.\n",
      "The topic of social features is completed with this chapter. In the next chapter, you\n",
      "will learn how to expose the application functionality as an API that clients such as\n",
      "smartphone apps can use.\n",
      "Comment Moderation \n",
      "| \n",
      "197\n",
      "\n",
      "\n",
      "CHAPTER 14\n",
      "Application Programming Interfaces\n",
      "In recent years, there has been a trend in web applications to move more and more of\n",
      "the business logic to the client side, producing an architecture that is known as Rich\n",
      "Internet Applications (RIAs). In RIAs, the server’s main (and sometimes only) func‐\n",
      "tion is to provide the client application with data retrieval and storage services. In this\n",
      "model, the server becomes a web service or application programming interface (API).\n",
      "There are several protocols by which RIAs can communicate with a web service.\n",
      "Remote procedure call (RPC) protocols such as XML-RPC or its derivative, the Sim‐\n",
      "plified Object Access Protocol (SOAP), were popular choices a few years ago. More\n",
      "recently, the Representational State Transfer (REST) architecture has emerged as the\n",
      "favorite for web applications due to its being built on the familiar model of the World\n",
      "Wide Web.\n",
      "Flask is an ideal framework to build RESTful web services, thanks to its lightweight\n",
      "nature. In this chapter, you will learn how to implement a Flask-based RESTful API.\n",
      "Introduction to REST\n",
      "Roy Fielding’s PhD dissertation describes the REST architectural style for web serv‐\n",
      "ices in terms of its six defining characteristics:\n",
      "Client–server\n",
      "There must be a clear separation between clients and servers.\n",
      "Stateless\n",
      "A client request must contain all the information that is necessary to carry it out.\n",
      "The server must not store any state about the client that persists from one request\n",
      "to the next.\n",
      "199\n",
      "\n",
      "Cache\n",
      "Responses from the server can be labeled as cacheable or noncacheable so that\n",
      "clients (or intermediaries between clients and servers) can use a cache for optimi‐\n",
      "zation purposes.\n",
      "Uniform interface\n",
      "The protocol by which clients access server resources must be consistent, well\n",
      "defined, and standardized. This is the most complex aspect of REST, covering the\n",
      "use of unique resource identifiers, resource representations, self-descriptive mes‐\n",
      "sages between client and server, and hypermedia.\n",
      "Layered system\n",
      "Proxy servers, caches, or gateways can be inserted between clients and servers as\n",
      "necessary to improve performance, reliability, and scalability.\n",
      "Code-on-demand\n",
      "Clients can optionally download code from the server to execute in their context.\n",
      "Resources Are Everything\n",
      "The concept of resources is core to the REST architectural style. In this context, a\n",
      "resource is an item of interest in the domain of the application. For example, in the\n",
      "blogging application, users, blog posts, and comments are all resources.\n",
      "Each resource must have a unique identifier that represents it. When working with\n",
      "HTTP, identifiers for resources are URLs. Continuing with the blogging example, a\n",
      "blog post could be represented by the URL /api/posts/12345, where 12345 is the iden‐\n",
      "tifier for a post, such as the post’s database primary key. The format or contents of the\n",
      "URL do not really matter; all that matters is that each resource URL uniquely identi‐\n",
      "fies a resource.\n",
      "A collection of all the resources in a class also has an assigned URL. The URL for the\n",
      "collection of blog posts could be /api/posts/ and the URL for the collection of all com‐\n",
      "ments could be /api/comments/.\n",
      "An API can also define collection URLs that represent logical subsets of all the\n",
      "resources in a class. For example, the collection of all comments in blog post 12345\n",
      "could be represented by the URL /api/posts/12345/comments/. It is common to define\n",
      "URLs that represent collections of resources with a trailing slash, as this gives them a\n",
      "“subdirectory” representation.\n",
      "Be aware that Flask applies special treatment to routes that end\n",
      "with a slash. If a client requests a URL without a trailing slash and\n",
      "there is a matching route that has a slash at the end, then Flask will\n",
      "automatically respond with a redirect to the trailing-slash URL. No\n",
      "redirects are issued for the reverse case.\n",
      "200 \n",
      "| \n",
      "Chapter 14: Application Programming Interfaces\n",
      "\n",
      "Request Methods\n",
      "The client application sends requests to the server at the established resource URLs\n",
      "and uses the request method to indicate the desired operation. To obtain the list of\n",
      "available blog posts in the blogging API the client would send a GET request to http://\n",
      "www.example.com/api/posts/, and to insert a new blog post it would send a POST\n",
      "request to the same URL, with the contents of the blog post in the request body. To\n",
      "retrieve blog post 12345 the client would send a GET request to http://www.exam‐\n",
      "ple.com/api/posts/12345. Table 14-1 lists the request methods that are commonly used\n",
      "in RESTful APIs, with their meanings.\n",
      "Table 14-1. HTTP request methods in RESTful APIs\n",
      "Request\n",
      "method\n",
      "Target\n",
      "Description\n",
      "HTTP response\n",
      "status code\n",
      "GET\n",
      "Individual resource URL\n",
      "Obtain the resource.\n",
      "200\n",
      "GET\n",
      "Resource collection URL Obtain the collection of resources (or one page from it if the\n",
      "server implements pagination).\n",
      "200\n",
      "POST\n",
      "Resource collection URL Create a new resource and add it to the collection. The server\n",
      "chooses the URL of the new resource and returns it in a\n",
      "Location header in the response.\n",
      "201\n",
      "PUT\n",
      "Individual resource URL\n",
      "Modify an existing resource. Alternatively, this method can also\n",
      "be used to create a new resource when the client can choose the\n",
      "resource URL.\n",
      "200 or 204\n",
      "DELETE\n",
      "Individual resource URL\n",
      "Delete a resource.\n",
      "200 or 204\n",
      "DELETE\n",
      "Resource collection URL Delete all resources in the collection.\n",
      "200 or 204\n",
      "The REST architecture does not require that all methods be imple‐\n",
      "mented for a resource. If the client invokes a method that is not\n",
      "supported for a given resource, then a response with the 405 status\n",
      "code (Method Not Allowed) should be returned. Flask handles this\n",
      "error automatically.\n",
      "The GET, POST, PUT, and DELETE request methods are not the only ones. The HTTP\n",
      "protocol relies on other methods, such as HEAD and OPTIONS, which are automatically\n",
      "implemented by Flask.\n",
      "Request and Response Bodies\n",
      "Resources are sent back and forth between client and server in the bodies of requests\n",
      "and responses, but REST does not specify the format to use to encode resources. The\n",
      "Content-Type header in requests and responses is used to indicate the format in\n",
      "which a resource is encoded in the body. The standard content negotiation mecha‐\n",
      "Introduction to REST \n",
      "| \n",
      "201\n",
      "\n",
      "nisms in the HTTP protocol can be used between client and server to agree on a for‐\n",
      "mat that both support.\n",
      "The two formats commonly used with RESTful web services are JavaScript Object\n",
      "Notation (JSON) and Extensible Markup Language (XML). For web-based RIAs,\n",
      "JSON is attractive due to being much more concise than XML, and because of its\n",
      "close ties to JavaScript, the client-side scripting language used by web browsers.\n",
      "Returning to the blog example API, a blog post resource could have the following\n",
      "JSON representation:\n",
      "{\n",
      "    \"self_url\": \"http://www.example.com/api/posts/12345\",\n",
      "    \"title\": \"Writing RESTful APIs in Python\",\n",
      "    \"author_url\": \"http://www.example.com/api/users/2\",\n",
      "    \"body\": \"... text of the article here ...\",\n",
      "    \"comments_url\": \"http://www.example.com/api/posts/12345/comments\"\n",
      "}\n",
      "Note how the url, author_url, and comments_url fields are fully qualified resource\n",
      "URLs. This is important because these URLs allow the client to discover new resour‐\n",
      "ces.\n",
      "In a well-designed RESTful API, the client knows a short list of top-level resource\n",
      "URLs and then discovers the rest from links included in responses, similar to how\n",
      "you can discover new web pages while browsing the web by clicking on links that\n",
      "appear in pages that you know about.\n",
      "Versioning\n",
      "In a traditional server-centric web application, the server has full control of the appli‐\n",
      "cation. When an application is updated, installing the new version on the server is\n",
      "enough to update all users because even the parts of the application that run in the\n",
      "user’s web browser are downloaded from the server.\n",
      "The situation with RIAs and web services is more complicated, because often clients\n",
      "are developed independently of the server—maybe even by different people. Consider\n",
      "the case of an application where the RESTful web service is used by a variety of clients\n",
      "including web browsers and native smartphone clients. The web browser client can\n",
      "be updated on the server at any time, but the smartphone apps cannot be updated by\n",
      "force; the smartphone owner needs to allow the update to happen. Even if the smart‐\n",
      "phone owner is willing to update, it is not possible to orchestrate the upgrade of all\n",
      "existing instances of smartphone applications to coincide exactly with the deploy‐\n",
      "ment of the new server version.\n",
      "For these reasons, web services need to be more tolerant than regular web applica‐\n",
      "tions and be able to work with old versions of their clients. Changes to a web service\n",
      "must be done with extreme care, because backward-incompatible changes can cause\n",
      "202 \n",
      "| \n",
      "Chapter 14: Application Programming Interfaces\n",
      "\n",
      "existing clients to break until they are upgraded. A common practice is to give web\n",
      "services a version, which is added to all URLs defined in that version of the server\n",
      "application. For example, the first release of the blogging web service could expose\n",
      "the collection of blog posts at /api/v1/posts/.\n",
      "Including the web service version in the URL helps keep old and new features organ‐\n",
      "ized so that the server can provide new features to new clients while continuing to\n",
      "support old clients. An update to the blogging service could change the JSON format\n",
      "of blog posts and now expose blog posts as /api/v2/posts/, while keeping the older\n",
      "JSON format for clients that connect to /api/v1/posts/.\n",
      "Although supporting multiple versions of the server can become a maintenance bur‐\n",
      "den, there are situations in which this is the only way to allow the application to grow\n",
      "without causing problems to existing deployments. Older service versions can be\n",
      "deprecated and later removed, once all clients have migrated to a newer version.\n",
      "RESTful Web Services with Flask\n",
      "Flask makes it very easy to create RESTful web services. The familiar route() decora‐\n",
      "tor along with its methods optional argument can be used to declare the routes that\n",
      "handle the resource URLs exposed by the service. Working with JSON data is also\n",
      "simple, as JSON data included with a request can be obtained in dictionary format by\n",
      "calling request.get_json(), and a response that needs to contain JSON can be easily\n",
      "generated from a Python dictionary using Flask’s jsonify() helper function.\n",
      "The following sections show how Flasky can be extended with a RESTful web service\n",
      "that gives clients access to blog posts and related resources.\n",
      "Creating an API Blueprint\n",
      "The routes associated with a RESTful API form a self-contained subset of the applica‐\n",
      "tion, so putting them in their own blueprint is the best way to keep them well organ‐\n",
      "ized. The general structure of the API blueprint within the application is shown in\n",
      "Example 14-1.\n",
      "Example 14-1. API blueprint structure\n",
      "|-flasky\n",
      "  |-app/\n",
      "    |-api\n",
      "      |-__init__.py\n",
      "      |-users.py\n",
      "      |-posts.py\n",
      "      |-comments.py\n",
      "      |-authentication.py\n",
      "      |-errors.py\n",
      "      |-decorators.py\n",
      "RESTful Web Services with Flask \n",
      "| \n",
      "203\n",
      "\n",
      "Note how the package used for the API includes a version number in its name. If in\n",
      "the future a backward-incompatible version of the API needs to be introduced, it can\n",
      "be added as a separate package with a different version number and both APIs can be\n",
      "included in the application.\n",
      "The API blueprint implements each resource in a separate module. Modules to take\n",
      "care of authentication and error handling and to provide custom decorators are also\n",
      "included. The blueprint constructor is shown in Example 14-2.\n",
      "Example 14-2. app/api/__init__.py: API blueprint creation\n",
      "from flask import Blueprint\n",
      "api = Blueprint('api', __name__)\n",
      "from . import authentication, posts, users, comments, errors\n",
      "The structure of the blueprint package constructor is similar to that of the other blue‐\n",
      "prints. Importing all the components of the blueprint is necessary so that routes and\n",
      "other handlers are registered. Since many of these modules need to import the api\n",
      "blueprint referenced here, the imports are done at the bottom to help prevent errors\n",
      "due to circular dependencies.\n",
      "The registration of the API blueprint is shown in Example 14-3.\n",
      "Example 14-3. app/init.py: API blueprint registration\n",
      "def create_app(config_name):\n",
      "    # ...\n",
      "    from .api import api as api_blueprint\n",
      "    app.register_blueprint(api_blueprint, url_prefix='/api/v1')\n",
      "    # ...\n",
      "The API blueprint is registered with a URL prefix, so that all its routes will have their\n",
      "URLs prefixed with /api/v1. Adding a prefix when registering the blueprint is a good\n",
      "idea because it eliminates the need to hardcode the version number in every blueprint\n",
      "route.\n",
      "Error Handling\n",
      "A RESTful web service informs the client of the status of a request by sending the\n",
      "appropriate HTTP status code in the response, plus any additional information in the\n",
      "response body. The typical status codes that a client can expect to see from a web ser‐\n",
      "vice are listed in Table 14-2.\n",
      "204 \n",
      "| \n",
      "Chapter 14: Application Programming Interfaces\n",
      "\n",
      "Table 14-2. HTTP response status codes typically returned by APIs\n",
      "HTTP\n",
      "status\n",
      "code\n",
      "Name\n",
      "Description\n",
      "200\n",
      "OK\n",
      "The request was completed successfully.\n",
      "201\n",
      "Created\n",
      "The request was completed successfully and a new resource was created as a result.\n",
      "202\n",
      "Accepted\n",
      "The request was accepted for processing, but it is still in progress and will run\n",
      "asynchronously.\n",
      "204\n",
      "No Content\n",
      "The request was completed successfully and there is no data to return in the response.\n",
      "400\n",
      "Bad Request\n",
      "The request is invalid or inconsistent.\n",
      "401\n",
      "Unauthorized\n",
      "The request does not include authentication information or the credentials provided are\n",
      "invalid.\n",
      "403\n",
      "Forbidden\n",
      "The authentication credentials sent with the request are insufficient for the request.\n",
      "404\n",
      "Not Found\n",
      "The resource referenced in the URL was not found.\n",
      "405\n",
      "Method Not Allowed The method requested is not supported for the given resource.\n",
      "500\n",
      "Internal Server Error\n",
      "An unexpected error occurred while processing the request.\n",
      "The handling of status codes 404 and 500 presents a small complication, in that these\n",
      "errors are normally generated by Flask on its own, and will return an HTML\n",
      "response. This can confuse an API client, which will likely expect all responses in\n",
      "JSON format.\n",
      "One way to generate appropriate responses for all clients is to make the error han‐\n",
      "dlers adapt their responses based on the format requested by the client, a technique\n",
      "called content negotiation. Example 14-4 shows an improved 404 error handler that\n",
      "responds with JSON to web service clients and with HTML to others. The 500 error\n",
      "handler is written in a similar way.\n",
      "Example 14-4. app/api/errors.py: 404 error handler with HTTP content negotiation\n",
      "@main.app_errorhandler(404)\n",
      "def page_not_found(e):\n",
      "    if request.accept_mimetypes.accept_json and \\\n",
      "            not request.accept_mimetypes.accept_html:\n",
      "        response = jsonify({'error': 'not found'})\n",
      "        response.status_code = 404\n",
      "        return response\n",
      "    return render_template('404.html'), 404\n",
      "This new version of the error handler checks the Accept request header, which is\n",
      "decoded into request.accept_mimetypes, to determine what format the client wants\n",
      "the response in. Browsers generally do not specify any restrictions on response for‐\n",
      "mats, but API clients typically do. The JSON response is generated only for clients\n",
      "that include JSON in their list of accepted formats, but not HTML.\n",
      "RESTful Web Services with Flask \n",
      "| \n",
      "205\n",
      "\n",
      "The remaining status codes are generated explicitly by the web service, so they can be\n",
      "implemented as helper functions inside the blueprint in the errors.py module.\n",
      "Example 14-5 shows the implementation of the 403 error; the others are similar.\n",
      "Example 14-5. app/api/errors.py: API error handler for status code 403\n",
      "def forbidden(message):\n",
      "    response = jsonify({'error': 'forbidden', 'message': message})\n",
      "    response.status_code = 403\n",
      "    return response\n",
      "View functions in the API blueprint can invoke these auxiliary functions to generate\n",
      "error responses when necessary.\n",
      "User Authentication with Flask-HTTPAuth\n",
      "Web services, like regular web applications, need to protect information and ensure\n",
      "that it is not given to unauthorized parties. For this reason, RIAs must ask their users\n",
      "for login credentials and pass them to the server for verification.\n",
      "It was mentioned earlier that one of the characteristics of RESTful web services is that\n",
      "they are stateless, which means that the server is not allowed to “remember” anything\n",
      "about the client between requests. Clients need to provide all the information neces‐\n",
      "sary to carry out a request in the request itself, so all requests must include user cre‐\n",
      "dentials.\n",
      "The current login functionality implemented with the help of Flask-Login stores data\n",
      "in the user session, which Flask stores by default in a client-side cookie, so the server\n",
      "does not store any user-related information; it asks the client to store it instead. It\n",
      "would appear that this implementation complies with the stateless requirement of\n",
      "REST, but the use of cookies in RESTful web services falls into a gray area, as it can be\n",
      "cumbersome for clients that are not web browsers to implement them. For that rea‐\n",
      "son, it is generally seen as a bad design choice to use cookies in APIs.\n",
      "The stateless requirement of REST may seem overly strict, but it is\n",
      "not arbitrary. Stateless servers can scale very easily. If servers store\n",
      "information about clients, it is necessary to ensure that the same\n",
      "server always gets requests from a given client, or else to use shared\n",
      "storage for client data. Both are complex problems to solve that do\n",
      "not exist when the server is stateless.\n",
      "Because the RESTful architecture is based on the HTTP protocol, HTTP authentica‐\n",
      "tion is the preferred method used to send credentials, either in its Basic or Digest fla‐\n",
      "vor. With HTTP authentication, user credentials are included in an Authorization\n",
      "header with all requests.\n",
      "206 \n",
      "| \n",
      "Chapter 14: Application Programming Interfaces\n",
      "\n",
      "The HTTP authentication protocol is simple enough that it can be implemented\n",
      "directly, but the Flask-HTTPAuth extension provides a convenient wrapper that hides\n",
      "the protocol details in a decorator similar to Flask-Login’s login_required.\n",
      "Flask-HTTPAuth is installed with pip:\n",
      "(venv) $ pip install flask-httpauth\n",
      "To initialize the extension for HTTP Basic authentication, an object of class\n",
      "HTTPBasicAuth must be created. Like Flask-Login, Flask-HTTPAuth makes no\n",
      "assumptions about the procedure required to verify user credentials, so this informa‐\n",
      "tion is given in a callback function. Example 14-6 shows how the extension is initial‐\n",
      "ized and provided with a verification callback.\n",
      "Example 14-6. app/api/authentication.py: Flask-HTTPAuth initialization\n",
      "from flask_httpauth import HTTPBasicAuth\n",
      "auth = HTTPBasicAuth()\n",
      "@auth.verify_password\n",
      "def verify_password(email, password):\n",
      "    if email == '':\n",
      "        return False\n",
      "    user = User.query.filter_by(email = email).first()\n",
      "    if not user:\n",
      "        return False\n",
      "    g.current_user = user\n",
      "    return user.verify_password(password)\n",
      "Because this type of user authentication will be used only in the API blueprint, the\n",
      "Flask-HTTPAuth extension is initialized in the blueprint package, and not in the\n",
      "application package like other extensions.\n",
      "The email and password are verified using the existing support in the User model.\n",
      "The verification callback returns True when the login is valid and False otherwise.\n",
      "The Flask-HTTPAuth extension also will invoke the callback for requests that carry\n",
      "no authentication, setting both arguments to the empty string. In this case, when\n",
      "email is an empty string, the function immediately returns False to block the\n",
      "request; for certain applications it may be acceptable to allow the anonymous user by\n",
      "returning True. The authentication callback saves the authenticated user in Flask’s g\n",
      "context variable so that the view function can access it later.\n",
      "Because user credentials are being exchanged with every request, it\n",
      "is extremely important that the API routes are exposed over secure\n",
      "HTTP so that all requests and responses are encrypted in transit.\n",
      "RESTful Web Services with Flask \n",
      "| \n",
      "207\n",
      "\n",
      "When the authentication credentials are invalid, the server returns a 401 status code\n",
      "response to the client. Flask-HTTPAuth generates a response with this status code by\n",
      "default, but to ensure that the response is consistent with other errors returned by the\n",
      "API, the error response can be customized as shown in Example 14-7.\n",
      "Example 14-7. _app/api/authentication.py: Flask-HTTPAuth error handler\n",
      "from .errors import unauthorized\n",
      "@auth.error_handler\n",
      "def auth_error():\n",
      "    return unauthorized('Invalid credentials')\n",
      "To protect a route, the auth.login_required decorator is used:\n",
      "@api.route('/posts/')\n",
      "@auth.login_required\n",
      "def get_posts():\n",
      "    pass\n",
      "But since all the routes in the blueprint need to be protected in the same way, the\n",
      "login_required decorator can be included once in a before_request handler for the\n",
      "blueprint, as shown in Example 14-8.\n",
      "Example 14-8. app/api/authentication.py: before_request handler with authentication\n",
      "from .errors import forbidden\n",
      "@api.before_request\n",
      "@auth.login_required\n",
      "def before_request():\n",
      "    if not g.current_user.is_anonymous and \\\n",
      "            not g.current_user.confirmed:\n",
      "        return forbidden('Unconfirmed account')\n",
      "Now the authentication checks will be done automatically for all the routes in the\n",
      "blueprint. As an additional check, the before_request handler also rejects authenti‐\n",
      "cated users who have not confirmed their accounts.\n",
      "Token-Based Authentication\n",
      "Clients must send authentication credentials with every request. To avoid having to\n",
      "constantly transfer sensitive information such as a password, a token-based authenti‐\n",
      "cation solution can be used.\n",
      "In token-based authentication, the client requests an access token by sending a\n",
      "request that includes the login credentials as authentication. The token can then be\n",
      "used in place of the login credentials to authenticate requests. For security reasons,\n",
      "208 \n",
      "| \n",
      "Chapter 14: Application Programming Interfaces\n",
      "\n",
      "tokens are issued with an associated expiration. When a token expires, the client must\n",
      "reauthenticate to get a new one. The risk of a token getting into the wrong hands is\n",
      "limited due to its short lifespan. Example 14-9 shows the two new methods added to\n",
      "the User model that support generation and verification of authentication tokens\n",
      "using itsdangerous.\n",
      "Example 14-9. app/models.py: token-based authentication support\n",
      "class User(db.Model):\n",
      "    # ...\n",
      "    def generate_auth_token(self, expiration):\n",
      "        s = Serializer(current_app.config['SECRET_KEY'],\n",
      "                       expires_in=expiration)\n",
      "        return s.dumps({'id': self.id}).decode('utf-8')\n",
      "    @staticmethod\n",
      "    def verify_auth_token(token):\n",
      "        s = Serializer(current_app.config['SECRET_KEY'])\n",
      "        try:\n",
      "            data = s.loads(token)\n",
      "        except:\n",
      "            return None\n",
      "        return User.query.get(data['id'])\n",
      "The generate_auth_token() method returns a signed token that encodes the user’s\n",
      "id field. An expiration time given in seconds is also used. The verify_auth_token()\n",
      "method takes a token and, if it’s found to be valid, returns the user stored in it. This is\n",
      "a static method, as the user will be known only after the token is decoded.\n",
      "To authenticate requests that come with a token, the verify_password callback for\n",
      "Flask-HTTPAuth must be modified to accept tokens as well as regular credentials. \n",
      "The updated callback is shown in Example 14-10.\n",
      "Example 14-10. app/api/authentication.py: improved authentication verification with\n",
      "token support\n",
      "@auth.verify_password\n",
      "def verify_password(email_or_token, password):\n",
      "    if email_or_token == '':\n",
      "        return False\n",
      "    if password == '':\n",
      "        g.current_user = User.verify_auth_token(email_or_token)\n",
      "        g.token_used = True\n",
      "        return g.current_user is not None\n",
      "    user = User.query.filter_by(email=email_or_token).first()\n",
      "    if not user:\n",
      "        return False\n",
      "    g.current_user = user\n",
      "RESTful Web Services with Flask \n",
      "| \n",
      "209\n",
      "\n",
      "    g.token_used = False\n",
      "    return user.verify_password(password)\n",
      "In this new version, the first authentication argument can be the email address or an\n",
      "authentication token. If this field is blank, an anonymous user is assumed, as before.\n",
      "If the password is blank, then the email_or_token field is assumed to be a token and\n",
      "validated as such. If both fields are nonempty then regular email and password\n",
      "authentication is assumed. With this implementation, token-based authentication is\n",
      "optional; it is up to each client to use it or not. To give view functions the ability to\n",
      "distinguish between the two authentication methods a g.token_used variable is\n",
      "added.\n",
      "The route that returns authentication tokens to the client is also added to the API\n",
      "blueprint. The implementation is shown in Example 14-11.\n",
      "Example 14-11. app/api/authentication.py: authentication token generation\n",
      "@api.route('/tokens/', methods=['POST'])\n",
      "def get_token():\n",
      "    if g.current_user.is_anonymous or g.token_used:\n",
      "        return unauthorized('Invalid credentials')\n",
      "    return jsonify({'token': g.current_user.generate_auth_token(\n",
      "        expiration=3600), 'expiration': 3600})\n",
      "Since this route is in the blueprint, the authentication mechanisms added to the\n",
      "before_request handler also apply to it. To prevent clients from authenticating to\n",
      "this route using a previously obtained token instead of an email address and pass‐\n",
      "word, the g.token_used variable is checked, and requests authenticated with a token\n",
      "are rejected. The purpose of this is to prevent users from bypassing the token expira‐\n",
      "tion by requesting a new token using the old token as authentication. The function\n",
      "returns a token in the JSON response with a validity period of one hour. The period is\n",
      "also included in the JSON response.\n",
      "Serializing Resources to and from JSON\n",
      "A frequent need when writing a web service is to convert internal representations of\n",
      "resources to and from JSON, which is the transport format used in HTTP requests\n",
      "and responses. The process of converting an internal representation to a transport\n",
      "format such as JSON is called serialization. Example 14-12 shows a new to_json()\n",
      "method added to the Post class.\n",
      "210 \n",
      "| \n",
      "Chapter 14: Application Programming Interfaces\n",
      "\n",
      "Example 14-12. app/models.py: converting a post to a JSON serializable dictionary\n",
      "class Post(db.Model):\n",
      "    # ...\n",
      "    def to_json(self):\n",
      "        json_post = {\n",
      "            'url': url_for('api.get_post', id=self.id),\n",
      "            'body': self.body,\n",
      "            'body_html': self.body_html,\n",
      "            'timestamp': self.timestamp,\n",
      "            'author_url': url_for('api.get_user', id=self.author_id),\n",
      "            'comments_url': url_for('api.get_post_comments', id=self.id),\n",
      "            'comment_count': self.comments.count()\n",
      "        }\n",
      "        return json_post\n",
      "The url, author_url, and comments_url fields need to return the URLs for the\n",
      "respective resources, so these are generated with url_for() calls to other routes that\n",
      "will be defined in the API blueprint.\n",
      "This example shows how it is possible to return “made-up” attributes in the represen‐\n",
      "tation of a resource. The comment_count field returns the number of comments that\n",
      "exist for the blog post. Although this is not a real attribute of the model, it is included\n",
      "in the resource representation as a convenience to the client.\n",
      "The to_json() method for User models can be constructed in a similar way. This\n",
      "method is shown in Example 14-13.\n",
      "Example 14-13. app/models.py: converting a user to a JSON serializable dictionary\n",
      "class User(UserMixin, db.Model):\n",
      "    # ...\n",
      "    def to_json(self):\n",
      "        json_user = {\n",
      "            'url': url_for('api.get_user', id=self.id),\n",
      "            'username': self.username,\n",
      "            'member_since': self.member_since,\n",
      "            'last_seen': self.last_seen,\n",
      "            'posts_url': url_for('api.get_user_posts', id=self.id),\n",
      "            'followed_posts_url': url_for('api.get_user_followed_posts',\n",
      "                                          id=self.id),\n",
      "            'post_count': self.posts.count()\n",
      "        }\n",
      "        return json_user\n",
      "Note how in this method some of the attributes of the user, such as email and role,\n",
      "are omitted from the response for privacy reasons. This example again demonstrates\n",
      "RESTful Web Services with Flask \n",
      "| \n",
      "211\n",
      "\n",
      "that the representation of a resource offered to clients does not need to be identical to\n",
      "the internal definition of the corresponding database model.\n",
      "The inverse of serialization is called deserialization. Deserializing a JSON structure\n",
      "back to a model presents the challenge that some of the data coming from the client\n",
      "might be invalid, wrong, or unnecessary. Example 14-14 shows the method that cre‐\n",
      "ates a Post from JSON.\n",
      "Example 14-14. app/models.py: creating a blog post from JSON\n",
      "from app.exceptions import ValidationError\n",
      "class Post(db.Model):\n",
      "    # ...\n",
      "    @staticmethod\n",
      "    def from_json(json_post):\n",
      "        body = json_post.get('body')\n",
      "        if body is None or body == '':\n",
      "            raise ValidationError('post does not have a body')\n",
      "        return Post(body=body)\n",
      "As you can see, this implementation chooses to only use the body attribute from the\n",
      "JSON dictionary. The body_html attribute is ignored since the server-side Markdown\n",
      "rendering is automatically triggered by an SQLAlchemy event whenever the body\n",
      "attribute is modified. The timestamp attribute does not need to be given unless the\n",
      "client is allowed to back- or future-date posts, which is not a feature this application\n",
      "supports. The author_url field is not used because the client has no authority to\n",
      "select the author of a blog post; the only possible value for this field is that of the\n",
      "authenticated user. The comments_url and comment_count attributes are automati‐\n",
      "cally generated from a database relationship, so there is no useful information in\n",
      "them that is needed to create a Post. Finally, the url field is ignored because in this\n",
      "implementation the resource URLs are defined by the server, not the client.\n",
      "Note how error checking is done. If the body field is missing or empty then a\n",
      "ValidationError exception is raised. Raising an exception is in this case the appro‐\n",
      "priate way to deal with the error because this method does not have enough knowl‐\n",
      "edge to properly handle the error condition. The exception effectively passes the\n",
      "error up to the caller, enabling higher-level code to do the error handling. The\n",
      "ValidationError class is implemented as a simple subclass of Python’s ValueError.\n",
      "This implementation is shown in Example 14-15.\n",
      "Example 14-15. app/exceptions.py: ValidationError exception\n",
      "class ValidationError(ValueError):\n",
      "    pass\n",
      "212 \n",
      "| \n",
      "Chapter 14: Application Programming Interfaces\n",
      "\n",
      "The application now needs to handle this exception by providing the appropriate\n",
      "response to the client. To avoid having to add exception-catching code in view func‐\n",
      "tions, a global exception handler can be installed using Flask’s errorhandler decora‐\n",
      "tor. A handler for the ValidationError exception is shown in Example 14-16.\n",
      "Example 14-16. app/api/errors.py: API error handler for ValidationError exceptions\n",
      "@api.errorhandler(ValidationError)\n",
      "def validation_error(e):\n",
      "    return bad_request(e.args[0])\n",
      "The errorhandler decorator is the same one that is used to register handlers for\n",
      "HTTP status codes, but in this usage it takes an Exception class as an argument. The\n",
      "decorated function will be invoked any time an exception of the given class is raised.\n",
      "Note that the decorator is obtained from the API blueprint, so this handler will be\n",
      "invoked only when the exception is raised while a route from the blueprint is being\n",
      "handled. Using this technique, the code in view functions can be written very cleanly\n",
      "and concisely, without the need to include error checking. For example:\n",
      "@api.route('/posts/', methods=['POST'])\n",
      "def new_post():\n",
      "    post = Post.from_json(request.json)\n",
      "    post.author = g.current_user\n",
      "    db.session.add(post)\n",
      "    db.session.commit()\n",
      "    return jsonify(post.to_json())\n",
      "Implementing Resource Endpoints\n",
      "What remains is to implement the routes that handle the different resources. The GET\n",
      "requests are typically the easiest because they just return information and don’t need\n",
      "to make any changes. Example 14-17 shows the two GET handlers for blog posts.\n",
      "Example 14-17. app/api/posts.py: GET resource handlers for posts\n",
      "@api.route('/posts/')\n",
      "def get_posts():\n",
      "    posts = Post.query.all()\n",
      "    return jsonify({ 'posts': [post.to_json() for post in posts] })\n",
      "@api.route('/posts/<int:id>')\n",
      "def get_post(id):\n",
      "    post = Post.query.get_or_404(id)\n",
      "    return jsonify(post.to_json())\n",
      "The first route handles the request for the collection of posts. This function uses a list\n",
      "comprehension to generate the JSON version of all the posts. The second route\n",
      "RESTful Web Services with Flask \n",
      "| \n",
      "213\n",
      "\n",
      "returns a single blog post and responds with a code 404 error when the given id is\n",
      "not found in the database.\n",
      "The POST handler for blog post resources inserts a new blog post in the database. This\n",
      "route is shown in Example 14-18.\n",
      "Example 14-18. app/api/posts.py: POST resource handler for posts\n",
      "@api.route('/posts/', methods=['POST'])\n",
      "@permission_required(Permission.WRITE)\n",
      "def new_post():\n",
      "    post = Post.from_json(request.json)\n",
      "    post.author = g.current_user\n",
      "    db.session.add(post)\n",
      "    db.session.commit()\n",
      "    return jsonify(post.to_json()), 201, \\\n",
      "        {'Location': url_for('api.get_post', id=post.id)}\n",
      "This view function is wrapped in a permission_required decorator (shown in an\n",
      "upcoming example) that ensures that the authenticated user has the permission to\n",
      "write blog posts. The actual creation of the blog post is straightforward due to the\n",
      "error handling support that was implemented previously. A blog post is created from\n",
      "the JSON data and its author is explicitly assigned as the authenticated user. After the\n",
      "model is written to the database, a 201 status code is returned and a Location header\n",
      "is added with the URL of the newly created resource.\n",
      "Note that as a convenience to clients, the body of the response includes the new\n",
      "resource. This will save the client from having to issue a GET request for it immedi‐\n",
      "ately after creating the resource.\n",
      "The permission_required decorator used to prevent unauthorized users from creat‐\n",
      "ing new blog posts is similar to the one used in the application but is customized for\n",
      "the API blueprint. The implementation is shown in Example 14-19.\n",
      "Example 14-19. app/api/decorators.py: permission_required decorator\n",
      "def permission_required(permission):\n",
      "    def decorator(f):\n",
      "        @wraps(f)\n",
      "        def decorated_function(*args, **kwargs):\n",
      "            if not g.current_user.can(permission):\n",
      "                return forbidden('Insufficient permissions')\n",
      "            return f(*args, **kwargs)\n",
      "        return decorated_function\n",
      "    return decorator\n",
      "214 \n",
      "| \n",
      "Chapter 14: Application Programming Interfaces\n",
      "\n",
      "The PUT handler for blog posts, used for editing existing resources, is shown in\n",
      "Example 14-20.\n",
      "Example 14-20. app/api/posts.py: PUT resource handler for posts\n",
      "@api.route('/posts/<int:id>', methods=['PUT'])\n",
      "@permission_required(Permission.WRITE)\n",
      "def edit_post(id):\n",
      "    post = Post.query.get_or_404(id)\n",
      "    if g.current_user != post.author and \\\n",
      "            not g.current_user.can(Permission.ADMIN):\n",
      "        return forbidden('Insufficient permissions')\n",
      "    post.body = request.json.get('body', post.body)\n",
      "    db.session.add(post)\n",
      "    db.session.commit()\n",
      "    return jsonify(post.to_json())\n",
      "The permission checks are more complex in this case. The standard check for per‐\n",
      "mission to write blog posts is done with the decorator, but to allow a user to edit a\n",
      "blog post the function must also ensure that the user is the author of the post or else\n",
      "is an administrator. This check is added explicitly to the view function. If this check\n",
      "had to be added in many view functions, building a decorator for it would be a good\n",
      "way to avoid code repetition.\n",
      "Since the application does not allow deletion of posts, the handler for the DELETE\n",
      "request method does not need to be implemented.\n",
      "The resource handlers for users and comments are implemented in a similar way.\n",
      "Table 14-3 lists the set of resources implemented for this application and the HTTP\n",
      "methods each supports. The complete implementation is available for you to study in\n",
      "the GitHub repository for this application.\n",
      "Table 14-3. Flasky API resources\n",
      "Resource URL\n",
      "Method\n",
      "Description\n",
      "/users/<int:id>\n",
      "GET\n",
      "Return a user.\n",
      "/users/<int:id>/posts/\n",
      "GET\n",
      "Return all the blog posts written by a user.\n",
      "/users/<int:id>/timeline/\n",
      "GET\n",
      "Return all the blog posts followed by a user.\n",
      "/posts/\n",
      "GET\n",
      "Return all the blog posts.\n",
      "/posts/\n",
      "POST\n",
      "Create a new blog post.\n",
      "/posts/<int:id>\n",
      "GET\n",
      "Return a blog post.\n",
      "/posts/<int:id>\n",
      "PUT\n",
      "Modify a blog post.\n",
      "/posts/<int:id>/comments/\n",
      "GET\n",
      "Return the comments on a blog post.\n",
      "/posts/<int:id>/comments/\n",
      "POST\n",
      "Add a comment to a blog post.\n",
      "/comments/\n",
      "GET\n",
      "Return all the comments.\n",
      "RESTful Web Services with Flask \n",
      "| \n",
      "215\n",
      "\n",
      "Resource URL\n",
      "Method\n",
      "Description\n",
      "/comments/<int:id>\n",
      "GET\n",
      "Return a comment.\n",
      "Note that the resources that were implemented offer only a subset of the functionality\n",
      "that is available through the web application. The list of supported resources could be\n",
      "expanded if necessary, such as to expose followers, to enable comment moderation,\n",
      "and to implement any other features that an API client might need.\n",
      "Pagination of Large Resource Collections\n",
      "The GET requests that return a collection of resources can be extremely expensive and\n",
      "difficult to manage for very large collections. Like web applications, web services can\n",
      "choose to paginate collections.\n",
      "Example 14-21 shows a possible implementation of pagination for the list of blog\n",
      "posts.\n",
      "Example 14-21. app/api/posts.py: Post pagination\n",
      "@api.route('/posts/')\n",
      "def get_posts():\n",
      "    page = request.args.get('page', 1, type=int)\n",
      "    pagination = Post.query.paginate(\n",
      "        page, per_page=current_app.config['FLASKY_POSTS_PER_PAGE'],\n",
      "        error_out=False)\n",
      "    posts = pagination.items\n",
      "    prev = None\n",
      "    if pagination.has_prev:\n",
      "        prev = url_for('api.get_posts', page=page-1)\n",
      "    next = None\n",
      "    if pagination.has_next:\n",
      "        next = url_for('api.get_posts', page=page+1)\n",
      "    return jsonify({\n",
      "        'posts': [post.to_json() for post in posts],\n",
      "        'prev_url': prev,\n",
      "        'next_url': next,\n",
      "        'count': pagination.total\n",
      "    })\n",
      "The posts field in the JSON response contains the data items as before, but now it is\n",
      "just a page and not the complete set. The prev_url and next_url items contain the\n",
      "resource URLs for the previous and following pages, or None when a page in that\n",
      "direction is not available. The count value is the total number of items in the collec‐\n",
      "tion.\n",
      "This technique can be applied to all the routes that return collections.\n",
      "216 \n",
      "| \n",
      "Chapter 14: Application Programming Interfaces\n",
      "\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 14a to check out this version of the applica‐\n",
      "tion. To ensure that you have all the dependencies installed, also\n",
      "run pip install -r requirements/dev.txt.\n",
      "Testing Web Services with HTTPie\n",
      "To test a web service, an HTTP client must be used. The two most used clients for\n",
      "testing Python web services from the command line are cURL and HTTPie. While\n",
      "both are useful tools, the latter has a much more concise and readable command line\n",
      "syntax that is tailored specifically to API requests. HTTPie is installed with pip:\n",
      "(venv) $ pip install httpie\n",
      "Assuming the development server is running on the default http://127.0.0.1:5000\n",
      "address, a GET request can be issued from another terminal window as follows:\n",
      "(venv) $ http --json --auth <email>:<password> GET \\\n",
      "> http://127.0.0.1:5000/api/v1/posts\n",
      "HTTP/1.0 200 OK\n",
      "Content-Length: 7018\n",
      "Content-Type: application/json\n",
      "Date: Sun, 22 Dec 2013 08:11:24 GMT\n",
      "Server: Werkzeug/0.9.4 Python/2.7.3\n",
      "{\n",
      "    \"posts\": [\n",
      "        ...\n",
      "    ],\n",
      "    \"prev_url\": null\n",
      "    \"next_url\": \"http://127.0.0.1:5000/api/v1/posts/?page=2\",\n",
      "    \"count\": 150\n",
      "}\n",
      "Note the pagination links included in the response. Since this is the first page, a previ‐\n",
      "ous page is not defined, but a URL to obtain the next page and a total count were\n",
      "returned.\n",
      "The following command sends a POST request to add a new blog post:\n",
      "(venv) $ http --auth <email>:<password> --json POST \\ \n",
      "> http://127.0.0.1:5000/api/v1/posts/ \\ \n",
      "> \"body=I'm adding a post from the *command line*.\"\n",
      "HTTP/1.0 201 CREATED\n",
      "Content-Length: 360\n",
      "Content-Type: application/json\n",
      "Date: Sun, 22 Dec 2013 08:30:27 GMT\n",
      "Location: http://127.0.0.1:5000/api/v1/posts/111\n",
      "Server: Werkzeug/0.9.4 Python/2.7.3\n",
      "{\n",
      "RESTful Web Services with Flask \n",
      "| \n",
      "217\n",
      "\n",
      "    \"author\": \"http://127.0.0.1:5000/api/v1/users/1\",\n",
      "    \"body\": \"I'm adding a post from the *command line*.\",\n",
      "    \"body_html\": \"<p>I'm adding a post from the <em>command line</em>.</p>\",\n",
      "    \"comments\": \"http://127.0.0.1:5000/api/v1/posts/111/comments\",\n",
      "    \"comment_count\": 0,\n",
      "    \"timestamp\": \"Sun, 22 Dec 2013 08:30:27 GMT\",\n",
      "    \"url\": \"http://127.0.0.1:5000/api/v1/posts/111\"\n",
      "}\n",
      "To use authentication tokens instead of a username and password, a POST request\n",
      "to /api/v1/tokens/ is sent first:\n",
      "(venv) $ http --auth <email>:<password> --json POST \\\n",
      "> http://127.0.0.1:5000/api/v1/tokens/\n",
      "HTTP/1.0 200 OK\n",
      "Content-Length: 162\n",
      "Content-Type: application/json\n",
      "Date: Sat, 04 Jan 2014 08:38:47 GMT\n",
      "Server: Werkzeug/0.9.4 Python/3.3.3\n",
      "{\n",
      "    \"expiration\": 3600,\n",
      "    \"token\": \"eyJpYXQiOjEzODg4MjQ3MjcsImV4cCI6MTM4ODgyODMyNywiYWxnIjoiSFMy...\"\n",
      "}\n",
      "And now the returned token can be used to make calls into the API for the next hour\n",
      "by passing it along in the username field and leaving the password empty:\n",
      "(venv) $ http --json --auth eyJpYXQ...: GET http://127.0.0.1:5000/api/v1/posts/ \n",
      "When the token expires, requests will be returned with a code 401 error, indicating\n",
      "that a new token needs to be obtained.\n",
      "Congratulations! This chapter completes Part II, and with that the feature develop‐\n",
      "ment phase of Flasky is complete. The next step is obviously to deploy it, and that\n",
      "brings a new set of challenges that are the subject of Part III.\n",
      "218 \n",
      "| \n",
      "Chapter 14: Application Programming Interfaces\n",
      "\n",
      "PART III\n",
      "The Last Mile\n",
      "\n",
      "\n",
      "CHAPTER 15\n",
      "Testing\n",
      "There are two very good reasons for writing unit tests. When implementing new\n",
      "functionality, unit tests are used to confirm that the new code is working in the\n",
      "expected way. The same result can be obtained by testing manually, but of course\n",
      "automated tests save time and effort because they can be repeated easily.\n",
      "A second, more important reason is that each time the application is modified, all the\n",
      "unit tests built around it can be executed to ensure that there are no regressions in the\n",
      "existing code; in other words, that the new changes did not affect the way the older\n",
      "code works.\n",
      "Unit tests have been a part of Flasky since the very beginning, with tests designed to\n",
      "exercise specific features of the application implemented in the database model\n",
      "classes. These classes are easy to test outside of the context of a running application,\n",
      "so given that it takes little effort, implementing unit tests for all the features that exist\n",
      "in the database models is the best way to ensure at least that part of the application\n",
      "starts robust and stays that way.\n",
      "This chapter discusses ways to improve and extend unit testing to other areas of the\n",
      "application.\n",
      "Obtaining Code Coverage Reports\n",
      "Having a test suite is important, but it is equally important to know how good or bad\n",
      "it is. Code coverage tools measure how much of the application is exercised by unit\n",
      "tests and can provide a detailed report that indicates which parts of the application\n",
      "code are not being tested. This information is invaluable, because it can be used to\n",
      "direct the effort of writing new tests to the areas that need it most.\n",
      "221\n",
      "\n",
      "Python has an excellent code coverage tool appropriately called coverage. You can\n",
      "install it with pip:\n",
      "(venv) $ pip install coverage\n",
      "This tool comes as a command-line script that can launch any Python application\n",
      "with code coverage enabled, but it also provides more convenient scripting access to\n",
      "start the coverage engine programmatically. To have coverage metrics nicely integra‐\n",
      "ted into the flask test command added in Chapter 7, a --coverage option can be\n",
      "added. The implementation of this option is shown in Example 15-1.\n",
      "Example 15-1. flasky.py: coverage metrics\n",
      "import os\n",
      "import sys\n",
      "import click\n",
      "COV = None\n",
      "if os.environ.get('FLASK_COVERAGE'):\n",
      "    import coverage\n",
      "    COV = coverage.coverage(branch=True, include='app/*')\n",
      "    COV.start()\n",
      "# ...\n",
      "@app.cli.command()\n",
      "@click.option('--coverage/--no-coverage', default=False,\n",
      "              help='Run tests under code coverage.')\n",
      "def test(coverage):\n",
      "    \"\"\"Run the unit tests.\"\"\"\n",
      "    if coverage and not os.environ.get('FLASK_COVERAGE'):\n",
      "        os.environ['FLASK_COVERAGE'] = '1'\n",
      "        os.execvp(sys.executable, [sys.executable] + sys.argv)\n",
      "    import unittest\n",
      "    tests = unittest.TestLoader().discover('tests')\n",
      "    unittest.TextTestRunner(verbosity=2).run(tests)\n",
      "    if COV:\n",
      "        COV.stop()\n",
      "        COV.save()\n",
      "        print('Coverage Summary:')\n",
      "        COV.report()\n",
      "        basedir = os.path.abspath(os.path.dirname(__file__))\n",
      "        covdir = os.path.join(basedir, 'tmp/coverage')\n",
      "        COV.html_report(directory=covdir)\n",
      "        print('HTML version: file://%s/index.html' % covdir)\n",
      "        COV.erase()\n",
      "The code coverage support is enabled by passing the --coverage option to the flask\n",
      "test command. To add the Boolean option to the test custom command, the\n",
      "222 \n",
      "| \n",
      "Chapter 15: Testing\n",
      "\n",
      "click.option decorator is used. Click then passes the value of the Boolean flag as an\n",
      "argument to the function.\n",
      "But integrating code coverage in the flasky.py script presents a small problem. By the\n",
      "time the --coverage option is received in the test() function, it is already too late to\n",
      "enable coverage metrics; by that time all the code in the global scope has already exe‐\n",
      "cuted. So, to get accurate metrics, the script recursively restarts itself after setting the\n",
      "FLASK_COVERAGE environment variable. In the second run, the top of the script finds\n",
      "that the environment variable is set and turns on coverage from the start, even before\n",
      "all the application imports.\n",
      "The coverage.coverage() function starts the coverage engine. The branch=True\n",
      "option enables branch coverage analysis, which, in addition to tracking which lines of\n",
      "code execute, checks whether for every conditional both the True and False cases\n",
      "have executed. The include option is used to limit coverage analysis to the files that\n",
      "are inside the application package, which is the only code that needs to be measured.\n",
      "Without the include option, all the extensions installed in the virtual environment\n",
      "and the code for the tests itself would be included in the coverage reports—and that\n",
      "would add a lot of noise to the report.\n",
      "After all the tests have executed, the test() function writes a report to the console\n",
      "and also writes a nicer HTML report to disk. The HTML version shows all the source\n",
      "code annotated with colors that indicate the lines that are covered by the tests and the\n",
      "ones that are not.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 15a to check out this version of the applica‐\n",
      "tion. To ensure that you have all the dependencies installed, also\n",
      "run pip install -r requirements/dev.txt.\n",
      "An example of the text-based report follows:\n",
      "(venv) $ flask test --coverage\n",
      "...\n",
      ".----------------------------------------------------------------------\n",
      "Ran 23 tests in 6.337s\n",
      "OK\n",
      "Coverage Summary:\n",
      "Name                           Stmts   Miss Branch BrPart  Cover\n",
      "----------------------------------------------------------------\n",
      "app/__init__.py                   32      0      0      0   100%\n",
      "app/api_v1/__init__.py             3      0      0      0   100%\n",
      "app/api_v1/authentication.py      29     18     10      0    28%\n",
      "app/api_v1/comments.py            40     30     12      0    19%\n",
      "app/api_v1/decorators.py          11      3      2      0    62%\n",
      "Obtaining Code Coverage Reports \n",
      "| \n",
      "223\n",
      "\n",
      "app/api_v1/errors.py              17     10      0      0    41%\n",
      "app/api_v1/posts.py               36     24      8      0    27%\n",
      "app/api_v1/users.py               30     24     12      0    14%\n",
      "app/auth/__init__.py               3      0      0      0   100%\n",
      "app/auth/forms.py                 45      8      8      0    70%\n",
      "app/auth/views.py                116     91     42      0    16%\n",
      "app/decorators.py                 14      3      2      0    69%\n",
      "app/email.py                      15      9      0      0    40%\n",
      "app/exceptions.py                  2      0      0      0   100%\n",
      "app/main/__init__.py               6      1      0      0    83%\n",
      "app/main/errors.py                20     15      6      0    19%\n",
      "app/main/forms.py                 39      7      6      0    71%\n",
      "app/main/views.py                178    140     34      0    18%\n",
      "app/models.py                    236     42     42      6    79%\n",
      "----------------------------------------------------------------\n",
      "TOTAL                            872    425    184      6    45%\n",
      "HTML version: file:///home/flask/flasky/tmp/coverage/index.html\n",
      "The report shows an overall coverage of 45%, which is not terrible, but isn’t very good\n",
      "either. The model classes, which have received all the unit testing attention so far,\n",
      "constitute a total of 236 statements, of which 79% are covered in tests. Obviously the\n",
      "views.py files in the main and auth blueprints and the routes in the api_v1 blueprint\n",
      "all have very low coverage, since these are not exercised in any of the existing unit\n",
      "tests. And of course, these coverage metrics are not indicative of how much bug-free\n",
      "code exists in the project, since other factors (such as the quality of the tests) play a\n",
      "big role in that.\n",
      "Armed with this report, it is easy to determine where tests need to be added to the\n",
      "test suite to improve coverage—but unfortunately, not all parts of the application can\n",
      "be tested as easily as the database models. The next two sections discuss more\n",
      "advanced testing strategies that can be applied to view functions, forms, and tem‐\n",
      "plates.\n",
      "The Flask Test Client\n",
      "Some portions of the application code rely heavily on the environment that is created\n",
      "by a running application. For example, you can’t simply invoke the code in a view\n",
      "function to test it, since the function may need to access Flask context variables such\n",
      "as request or session, it may be expecting form data provided in a POST request, and\n",
      "it may also require a logged-in user. In short, view functions can run only within the\n",
      "context of a request and a running application.\n",
      "Flask comes equipped with a test client to try to address this problem, at least to some\n",
      "extent. The test client replicates the environment that exists when an application is\n",
      "running inside a web server, allowing tests to act as clients and send requests.\n",
      "The view functions do not see any major differences when executed under the test\n",
      "client; requests are received and routed to the appropriate view functions, from which\n",
      "224 \n",
      "| \n",
      "Chapter 15: Testing\n",
      "\n",
      "responses are generated and returned. After a view function executes, its response is\n",
      "passed to the test, which can check it for correctness.\n",
      "Testing Web Applications\n",
      "Example 15-2 shows a unit testing framework that uses the test client.\n",
      "Example 15-2. tests/test_client.py: framework for tests using the Flask test client\n",
      "import unittest\n",
      "from app import create_app, db\n",
      "from app.models import User, Role\n",
      "class FlaskClientTestCase(unittest.TestCase):\n",
      "    def setUp(self):\n",
      "        self.app = create_app('testing')\n",
      "        self.app_context = self.app.app_context()\n",
      "        self.app_context.push()\n",
      "        db.create_all()\n",
      "        Role.insert_roles()\n",
      "        self.client = self.app.test_client(use_cookies=True)\n",
      "    def tearDown(self):\n",
      "        db.session.remove()\n",
      "        db.drop_all()\n",
      "        self.app_context.pop()\n",
      "    def test_home_page(self):\n",
      "        response = self.client.get('/')\n",
      "        self.assertEqual(response.status_code, 200)\n",
      "        self.assertTrue('Stranger' in response.get_data(as_text=True))\n",
      "Compared to tests/test_basics.py, this module adds a self.client instance variable,\n",
      "which is the Flask test client object. This object exposes methods that issue requests\n",
      "into the application. When the test client is created with the use_cookies option\n",
      "enabled, it will accept and send cookies in the same way browsers do, so functionality\n",
      "that relies on cookies to recall context between requests can be used. In particular,\n",
      "this approach enables the use of user sessions, which are stored in cookies.\n",
      "The test_home_page() test is a simple example of what the test client can do. In this\n",
      "example, a request for the root URL of the application is issued. The return value of\n",
      "the get() method of the test client is a Flask response object containing the response\n",
      "returned by the invoked view function. To check whether the test was successful, the\n",
      "status code of the response is checked, and then the body of the response, obtained\n",
      "from response.get_data(), is searched for the word \"Stranger\", which is part of\n",
      "the “Hello, Stranger!” greeting shown to anonymous users. Note that get_data()\n",
      "The Flask Test Client \n",
      "| \n",
      "225\n",
      "\n",
      "returns the response body as a byte array by default; passing as_text=True converts it\n",
      "to a string, which is easier to work with.\n",
      "The test client can also send POST requests that include form data using the post()\n",
      "method, but submitting forms presents a small complication. As discussed in Chap‐\n",
      "ter 4, all forms generated by Flask-WTF have a hidden field with a CSRF token that\n",
      "needs to be submitted along with the form. To be able to send the CSRF token, a test\n",
      "would need to request the page that displays the form, then parse the HTML returned\n",
      "in that response and extract the token, so that it can then send it with the form data.\n",
      "To avoid the hassle of dealing with CSRF tokens in tests, it is better to disable CSRF\n",
      "protection in the testing configuration. This is shown in Example 15-3.\n",
      "Example 15-3. config.py: disabling CSRF protection in the testing configuration\n",
      "class TestingConfig(Config):\n",
      "    #...\n",
      "    WTF_CSRF_ENABLED = False\n",
      "Example 15-4 shows a more advanced unit test that simulates a new user registering\n",
      "an account, logging in, confirming the account with a confirmation token, and finally\n",
      "logging out.\n",
      "Example 15-4. tests/test_client.py: simulation of a new user workflow with the Flask test\n",
      "client\n",
      "class FlaskClientTestCase(unittest.TestCase):\n",
      "    # ...\n",
      "    def test_register_and_login(self):\n",
      "        # register a new account\n",
      "        response = self.client.post('/auth/register', data={\n",
      "            'email': 'john@example.com',\n",
      "            'username': 'john',\n",
      "            'password': 'cat',\n",
      "            'password2': 'cat'\n",
      "        })\n",
      "        self.assertEqual(response.status_code, 302)\n",
      "        # log in with the new account\n",
      "        response = self.client.post('/auth/login', data={\n",
      "            'email': 'john@example.com',\n",
      "            'password': 'cat'\n",
      "        }, follow_redirects=True)\n",
      "        self.assertEqual(response.status_code, 200)\n",
      "        self.assertTrue(re.search('Hello,\\s+john!',\n",
      "                                  response.get_data(as_text=True)))\n",
      "        self.assertTrue(\n",
      "            'You have not confirmed your account yet' in response.get_data(\n",
      "                as_text=True))\n",
      "226 \n",
      "| \n",
      "Chapter 15: Testing\n",
      "\n",
      "        # send a confirmation token\n",
      "        user = User.query.filter_by(email='john@example.com').first()\n",
      "        token = user.generate_confirmation_token()\n",
      "        response = self.client.get('/auth/confirm/{}'.format(token),\n",
      "                                   follow_redirects=True)\n",
      "        user.confirm(token)\n",
      "        self.assertEqual(response.status_code, 200)\n",
      "        self.assertTrue(\n",
      "            'You have confirmed your account' in response.get_data(\n",
      "                as_text=True))\n",
      "        # log out\n",
      "        response = self.client.get('/auth/logout', follow_redirects=True)\n",
      "        self.assertEqual(response.status_code, 200)\n",
      "        self.assertTrue('You have been logged out' in response.get_data(\n",
      "            as_text=True))\n",
      "The test begins with a form submission to the registration route. The data argument\n",
      "to post() is a dictionary with the form fields, which must exactly match the field\n",
      "names defined in the HTML form. Since CSRF protection is now disabled in the test‐\n",
      "ing configuration, there is no need to send the CSRF token with the form.\n",
      "The /auth/register route can respond in two ways. If the registration data is valid, a\n",
      "redirect sends the user to the login page. In the case of an invalid registration, the\n",
      "response renders the page with the registration form again, including any appropriate\n",
      "error messages. To validate that the registration was accepted, the test checks that the\n",
      "status code of the response is 302, which is the code for a redirect.\n",
      "The second section of the test issues a login request to the application using the email\n",
      "and password just registered. This is done with a POST request to the /auth/login\n",
      "route. This time a follow_redirects=True argument is included in the post() call to\n",
      "make the test client work like a browser and automatically issue a GET request for the\n",
      "redirected URL. With this option, status code 302 will not be returned; instead, the\n",
      "response from the redirected URL is returned.\n",
      "A successful response to the login submission would now have a page that greets the\n",
      "user by their username and then indicates that the account needs to be confirmed to\n",
      "gain access. Two assert statements verify that this is the page returned. Here, it is\n",
      "interesting to note that a search for the string 'Hello, john!' would not work\n",
      "because this string is assembled from static and dynamic portions, so due to the way\n",
      "the Jinja2 template was created the final HTML has extra whitespace in between these\n",
      "two words. To avoid an error in this test due to the whitespace, a regular expression is\n",
      "used.\n",
      "The next step is to confirm the account, which presents another small obstacle. The\n",
      "confirmation URL is sent to the user by email during registration, so there is no easy\n",
      "way to access it from the test. The solution presented in the test bypasses the token\n",
      "The Flask Test Client \n",
      "| \n",
      "227\n",
      "\n",
      "that was generated as part of the registration and generates another one directly from\n",
      "the User instance. Another possibility would have been to extract the token by pars‐\n",
      "ing the email body, which Flask-Mail saves when running in a testing configuration.\n",
      "With the token at hand, the next step of the test is to simulate the user clicking the\n",
      "confirmation token URL received by email. This is achieved by sending a GET request\n",
      "to the confirmation URL, which includes the token. The response to this request is a\n",
      "redirect to the home page, but once again follow_redirects=True is specified, so the\n",
      "test client requests the redirected page automatically and returns it. The response is\n",
      "checked for the greeting and a flashed message that informs the user that the confir‐\n",
      "mation was successful.\n",
      "The final step in this test is to send a GET request to the logout route; to confirm that\n",
      "this has worked, the test searches for the flashed message in the response.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 15b to check out this version of the applica‐\n",
      "tion.\n",
      "Testing Web Services\n",
      "The Flask test client can also be used to test RESTful web services. Example 15-5\n",
      "shows an example unit test class with two tests.\n",
      "Example 15-5. tests/test_api.py: RESTful API testing with the Flask test client\n",
      "class APITestCase(unittest.TestCase):\n",
      "    # ...\n",
      "    def get_api_headers(self, username, password):\n",
      "        return {\n",
      "            'Authorization':\n",
      "                'Basic ' + b64encode(\n",
      "                    (username + ':' + password).encode('utf-8')).decode('utf-8'),\n",
      "            'Accept': 'application/json',\n",
      "            'Content-Type': 'application/json'\n",
      "        }\n",
      "    def test_no_auth(self):\n",
      "        response = self.client.get(url_for('api.get_posts'),\n",
      "                                   content_type='application/json')\n",
      "        self.assertEqual(response.status_code, 401)\n",
      "    def test_posts(self):\n",
      "        # add a user\n",
      "        r = Role.query.filter_by(name='User').first()\n",
      "        self.assertIsNotNone(r)\n",
      "228 \n",
      "| \n",
      "Chapter 15: Testing\n",
      "\n",
      "        u = User(email='john@example.com', password='cat', confirmed=True,\n",
      "                 role=r)\n",
      "        db.session.add(u)\n",
      "        db.session.commit()\n",
      "       # write a post\n",
      "        response = self.client.post(\n",
      "            '/api/v1/posts/',\n",
      "            headers=self.get_api_headers('john@example.com', 'cat'),\n",
      "            data=json.dumps({'body': 'body of the *blog* post'}))\n",
      "        self.assertEqual(response.status_code, 201)\n",
      "        url = response.headers.get('Location')\n",
      "        self.assertIsNotNone(url)\n",
      "        # get the new post\n",
      "        response = self.client.get(\n",
      "            url,\n",
      "            headers=self.get_api_headers('john@example.com', 'cat'))\n",
      "        self.assertEqual(response.status_code, 200)\n",
      "        json_response = json.loads(response.get_data(as_text=True))\n",
      "        self.assertEqual('http://localhost' + json_response['url'], url)\n",
      "        self.assertEqual(json_response['body'], 'body of the *blog* post')\n",
      "        self.assertEqual(json_response['body_html'],\n",
      "                        '<p>body of the <em>blog</em> post</p>')\n",
      "The setUp() and tearDown() methods for testing the API are the same as for the reg‐\n",
      "ular application, but the cookie support does not need to be configured because the\n",
      "API does not use it. The get_api_headers() method is a helper method that returns\n",
      "the common headers that need to be sent with most API requests. These include the\n",
      "authentication credentials and the MIME type-related headers.\n",
      "The test_no_auth() test is a simple test that ensures that a request that does not\n",
      "include authentication credentials is rejected with error code 401. The test_posts()\n",
      "test adds a user to the database and then uses the RESTful API to insert a blog post\n",
      "and then read it back. Any requests that send data in the body must encode it with\n",
      "json.dumps(), because the Flask test client does not automatically encode to JSON.\n",
      "Likewise, response bodies are also returned in JSON format and must be decoded\n",
      "with json.loads() before they can be inspected.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 15c to check out this version of the applica‐\n",
      "tion.\n",
      "The Flask Test Client \n",
      "| \n",
      "229\n",
      "\n",
      "End-to-End Testing with Selenium\n",
      "The Flask test client cannot fully emulate the environment of a running application.\n",
      "For example, any application that relies on JavaScript code running in the client\n",
      "browser will not work, as the JavaScript code included in the responses will be\n",
      "returned to the test without being executed.\n",
      "When tests require the complete environment, there is no other choice than to use a\n",
      "real web browser connected to the application running on a real web server. Fortu‐\n",
      "nately, most web browsers can be automated. Selenium is a web browser automation\n",
      "tool that supports the most popular web browsers in the three major operating sys‐\n",
      "tems.\n",
      "The Python interface for Selenium is installed with pip:\n",
      "(venv) $ pip install selenium\n",
      "Selenium requires a driver for the desired web browser to be installed separately, in\n",
      "addition to the browser itself. There are drivers for all major web browsers, so an\n",
      "application could set up a sophisticated framework to test several browsers. For this\n",
      "application, however, only the Google Chrome web browser will be used for automa‐\n",
      "ted tests, with its corresponding driver, ChromeDriver. If you are using a macOS\n",
      "computer with the brew package installer, you can install ChromeDriver as follows:\n",
      "(venv) $ brew install chromedriver\n",
      "For Linux, Microsoft Windows, or a macOS computer without brew, you can down‐\n",
      "load a regular ChromeDriver installer from the ChromeDriver website.\n",
      "Testing with Selenium requires the application to be running inside a web server that\n",
      "is listening for real HTTP requests. The method that will be shown in this section\n",
      "starts the application with the development server in a background thread while the\n",
      "tests run on the main thread. Under the control of the tests, Selenium launches a web\n",
      "browser and makes it connect to the application to perform the required operations.\n",
      "A problem with this approach is that after all the tests have completed, the Flask\n",
      "server needs to be stopped, ideally in a graceful way, so that background tasks such as\n",
      "the code coverage engine can cleanly complete their work. The Werkzeug web server\n",
      "has a shutdown option, but because the server is running isolated in its own thread,\n",
      "the only way to ask the server to shut down is by sending a regular HTTP request.\n",
      "Example 15-6 shows the implementation of a server shutdown route.\n",
      "230 \n",
      "| \n",
      "Chapter 15: Testing\n",
      "\n",
      "Example 15-6. _app/main/views.py: server shutdown route\n",
      "@main.route('/shutdown')\n",
      "def server_shutdown():\n",
      "    if not current_app.testing:\n",
      "        abort(404)\n",
      "    shutdown = request.environ.get('werkzeug.server.shutdown')\n",
      "    if not shutdown:\n",
      "        abort(500)\n",
      "    shutdown()\n",
      "    return 'Shutting down...'\n",
      "The shutdown route will work only when the application is running in testing mode;\n",
      "invoking it in other configurations will return a 404 status code response. The actual\n",
      "shutdown procedure involves calling a shutdown function that Werkzeug exposes in\n",
      "the environment. After calling this function and returning from the request, the\n",
      "development web server will know that it needs to exit gracefully.\n",
      "Example 15-7 shows the layout of a test case that is configured to run tests with Sele‐\n",
      "nium.\n",
      "Example 15-7. tests/test_selenium.py: framework for tests using Selenium\n",
      "from selenium import webdriver\n",
      "class SeleniumTestCase(unittest.TestCase):\n",
      "    client = None\n",
      "    @classmethod\n",
      "    def setUpClass(cls):\n",
      "        # start Chrome\n",
      "        options = webdriver.ChromeOptions()\n",
      "        options.add_argument('headless')\n",
      "        try:\n",
      "            cls.client = webdriver.Chrome(chrome_options=options)\n",
      "        except:\n",
      "            pass\n",
      "        # skip these tests if the browser could not be started\n",
      "        if cls.client:\n",
      "            # create the application\n",
      "            cls.app = create_app('testing')\n",
      "            cls.app_context = cls.app.app_context()\n",
      "            cls.app_context.push()\n",
      "            # suppress logging to keep unittest output clean\n",
      "            import logging\n",
      "            logger = logging.getLogger('werkzeug')\n",
      "            logger.setLevel(\"ERROR\")\n",
      "End-to-End Testing with Selenium \n",
      "| \n",
      "231\n",
      "\n",
      "            # create the database and populate with some fake data\n",
      "            db.create_all()\n",
      "            Role.insert_roles()\n",
      "            fake.users(10)\n",
      "            fake.posts(10)\n",
      "            # add an administrator user\n",
      "            admin_role = Role.query.filter_by(permissions=0xff).first()\n",
      "            admin = User(email='john@example.com',\n",
      "                         username='john', password='cat',\n",
      "                         role=admin_role, confirmed=True)\n",
      "            db.session.add(admin)\n",
      "            db.session.commit()\n",
      "            # start the Flask server in a thread\n",
      "            cls.server_thread = threading.Thread(\n",
      "                target=cls.app.run, kwargs={'debug': 'false',\n",
      "                                            'use_reloader': False,\n",
      "                                            'use_debugger': False})\n",
      "            cls.server_thread.start()\n",
      "    @classmethod\n",
      "    def tearDownClass(cls):\n",
      "        if cls.client:\n",
      "            # stop the Flask server and the browser\n",
      "            cls.client.get('http://localhost:5000/shutdown')\n",
      "            cls.client.quit()\n",
      "            cls.server_thread.join()\n",
      "            # destroy database\n",
      "            db.drop_all()\n",
      "            db.session.remove()\n",
      "            # remove application context\n",
      "            cls.app_context.pop()\n",
      "    def setUp(self):\n",
      "        if not self.client:\n",
      "            self.skipTest('Web browser not available')\n",
      "    def tearDown(self):\n",
      "        pass\n",
      "The setUpClass() and tearDownClass() class methods are invoked before and after\n",
      "the tests in this class execute. The setup involves starting an instance of Chrome\n",
      "through Selenium’s webdriver API, and creating an application and a database with\n",
      "some initial fake data for tests to use. The application is started in a thread using the\n",
      "app.run() method. At the end the application receives a request to /shutdown, which\n",
      "causes the background thread to end. The browser is then closed and the test data‐\n",
      "base removed.\n",
      "232 \n",
      "| \n",
      "Chapter 15: Testing\n",
      "\n",
      "Before the Flask command-line interface based on Click was intro‐\n",
      "duced, you had to start the Flask development web server by call‐\n",
      "ing app.run() from the application’s main script, or else use a\n",
      "third-party extension such as Flask-Script. While using app.run()\n",
      "to start a server is now replaced with the flask run command, the\n",
      "app.run() method continues to be supported, and here you can\n",
      "see how it can still be useful for complex unit testing situations.\n",
      "Selenium supports many other web browsers besides Chrome.\n",
      "Consult the Selenium documentation if you wish to use another\n",
      "web browser or test additional browsers.\n",
      "The setUp() method that runs before each test skips tests if Selenium cannot start the\n",
      "web browser in the startUpClass() method. In Example 15-8 you can see an exam‐\n",
      "ple test built with Selenium.\n",
      "Example 15-8. tests/test_selenium.py: example Selenium unit test\n",
      "class SeleniumTestCase(unittest.TestCase):\n",
      "    # ...\n",
      "    def test_admin_home_page(self):\n",
      "        # navigate to home page\n",
      "        self.client.get('http://localhost:5000/')\n",
      "        self.assertTrue(re.search('Hello,\\s+Stranger!',\n",
      "                                  self.client.page_source))\n",
      "        # navigate to login page\n",
      "        self.client.find_element_by_link_text('Log In').click()\n",
      "        self.assertIn('<h1>Login</h1>', self.client.page_source)\n",
      "        # log in\n",
      "        self.client.find_element_by_name('email').\\\n",
      "            send_keys('john@example.com')\n",
      "        self.client.find_element_by_name('password').send_keys('cat')\n",
      "        self.client.find_element_by_name('submit').click()\n",
      "        self.assertTrue(re.search('Hello,\\s+john!', self.client.page_source))\n",
      "        # navigate to the user's profile page\n",
      "        self.client.find_element_by_link_text('Profile').click()\n",
      "        self.assertIn('<h1>john</h1>', self.client.page_source)\n",
      "This test logs in to the application using the administrator account that was created in\n",
      "setUpClass() and then opens the user’s profile page. Note how different the testing\n",
      "methodology is from the Flask test client. When testing with Selenium, tests send\n",
      "commands to the web browser and never interact with the application directly. The\n",
      "End-to-End Testing with Selenium \n",
      "| \n",
      "233\n",
      "\n",
      "commands closely match the actions that a real user would perform with a mouse or\n",
      "keyboard.\n",
      "The test begins with a call to get() with the home page of the application. In the\n",
      "browser, this causes the URL to be entered in the address bar. To verify this step, the\n",
      "page source is checked for the “Hello, Stranger!” greeting.\n",
      "To go to the sign-in page, the test looks for the “Log In” link using\n",
      "find_element_by_link_text() and then calls click() on it to trigger a real click in\n",
      "the browser. Selenium provides several find_element_by...() convenience methods\n",
      "that can search for elements within the HTML page in different ways.\n",
      "To log in to the application, the test locates the email and password form fields by\n",
      "their names using find_element_by_name() and then writes text into them with\n",
      "send_keys(). The form is submitted by calling click() on the submit button. The\n",
      "personalized greeting is checked to ensure that the login was successful and the\n",
      "browser is now on the home page.\n",
      "The final part of the test locates the “Profile” link in the navigation bar and clicks it.\n",
      "To verify that the profile page was loaded, the heading with the username is searched\n",
      "in the page source.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 15d to check out this version of the applica‐\n",
      "tion. This update contains a database migration, so remember to\n",
      "run flask db upgrade after you check out the code. To ensure that\n",
      "you have all the dependencies installed, also run pip install -r\n",
      "requirements/dev.txt.\n",
      "When you run the unit tests with the flask test command there will be no visible\n",
      "difference. The test_admin_home_page unit test in Example 15-8 will run a headless\n",
      "Chrome instance and perform all the actions on it. If you want to see the actions per‐\n",
      "formed \n",
      "in \n",
      "a \n",
      "real \n",
      "Chrome \n",
      "window, \n",
      "comment \n",
      "out \n",
      "the \n",
      "line\n",
      "options.add_argument('headless') in the setUpClass() method, so that Selenium\n",
      "creates a regular Chrome window.\n",
      "Is It Worth It?\n",
      "By now you may be asking yourself if testing using the Flask test client or Selenium is\n",
      "really worth the trouble. It is a valid question, and it does not have a simple answer.\n",
      "Whether you like it or not, your application will be tested. If you don’t test it yourself,\n",
      "then your users will become the unwilling testers; they will find the bugs, and then\n",
      "you will have to fix them under pressure. Simple and focused tests like the ones that\n",
      "exercise database models and other parts of the application that can be executed out‐\n",
      "234 \n",
      "| \n",
      "Chapter 15: Testing\n",
      "\n",
      "side of the context of an application should always be carried out, as they have a very\n",
      "low cost and ensure the proper functioning of the core pieces of application logic.\n",
      "End-to-end tests of the type that the Flask test client and Selenium can carry out are\n",
      "sometimes necessary, but due to the increased complexity of writing them, they\n",
      "should be used only for functionality that cannot be tested in isolation. The applica‐\n",
      "tion code should be organized so that it is possible to push the business logic into\n",
      "application modules that are independent of the context of the application, and thus\n",
      "can be tested more easily. The code that exists in view functions should be simple and\n",
      "just act as a thin layer that accepts requests and invokes the corresponding actions in\n",
      "other classes or functions that encapsulate the application logic.\n",
      "So yes, testing is absolutely worth it. But it is important to design an efficient testing\n",
      "strategy and write code that can take advantage of it.\n",
      "Is It Worth It? \n",
      "| \n",
      "235\n",
      "\n",
      "\n",
      "CHAPTER 16\n",
      "Performance\n",
      "Nobody likes slow applications. Long waits for pages to load frustrate users, so it is\n",
      "important to detect and correct performance problems as soon as they appear. In this\n",
      "chapter, two important performance aspects of web applications are considered.\n",
      "Logging Slow Database Performance\n",
      "When application performance slowly degenerates with time, it is likely due to slow\n",
      "database queries, which get worse as the size of the database grows. Optimizing data‐\n",
      "base queries can be as simple as adding more indexes or as complex as adding a cache\n",
      "between the application and the database. The explain statement, available in most\n",
      "database query languages, shows the steps the database takes to execute a given query,\n",
      "often exposing inefficiencies in database or index design.\n",
      "But before starting to optimize queries, it is necessary to determine which queries are\n",
      "the ones that are worth optimizing. During a typical request several database queries\n",
      "may be issued, so it is often hard to identify which of all the queries are the slow ones.\n",
      "Flask-SQLAlchemy has an option to record statistics about database queries issued\n",
      "during a request. In Example 16-1 you can see how this feature can be used to log\n",
      "queries that are slower than a configured threshold.\n",
      "Example 16-1. app/main/views.py: reporting slow database queries\n",
      "from flask_sqlalchemy import get_debug_queries\n",
      "@main.after_app_request\n",
      "def after_request(response):\n",
      "    for query in get_debug_queries():\n",
      "        if query.duration >= current_app.config['FLASKY_SLOW_DB_QUERY_TIME']:\n",
      "            current_app.logger.warning(\n",
      "237\n",
      "\n",
      "                'Slow query: %s\\nParameters: %s\\nDuration: %fs\\nContext: %s\\n' %\n",
      "                    (query.statement, query.parameters, query.duration,\n",
      "                     query.context))\n",
      "    return response\n",
      "This functionality is attached to an after_app_request handler, which works in a\n",
      "similar way to the before_app_request handler but is invoked after the view func‐\n",
      "tion that handles the request returns. Flask passes the response object to the\n",
      "after_app_request handler in case it needs to be modified.\n",
      "In this case, the after_app_request handler does not modify the response; it just\n",
      "gets the query timings recorded by Flask-SQLAlchemy and then logs the slow ones to\n",
      "the application logger that Flask sets up at app.logger, before returning the response,\n",
      "which will then be sent to the client.\n",
      "The get_debug_queries() function returns the queries issued during the request as a\n",
      "list. The information provided for each query is shown in Table 16-1.\n",
      "Table 16-1. Query statistics recorded by Flask-SQLAlchemy\n",
      "Name\n",
      "Description\n",
      "statement\n",
      "The SQL statement\n",
      "parameters\n",
      "The parameters used with the SQL statement\n",
      "start_time The time the query was issued\n",
      "end_time\n",
      "The time the query returned\n",
      "duration\n",
      "The duration of the query in seconds\n",
      "context\n",
      "A string that indicates the source code location where the query was issued\n",
      "The after_app_request handler walks the list and logs any queries that lasted longer\n",
      "than a threshold given in the configuration variable FLASKY_SLOW_DB_QUERY_TIME.\n",
      "The logging is issued at the warning level in this application, but in some cases it may\n",
      "make sense to treat slow database alerts as errors.\n",
      "The get_debug_queries() function is enabled only in debug mode by default.\n",
      "Unfortunately, database performance problems rarely show up during development\n",
      "because much smaller databases are used. For this reason, it is much more useful to\n",
      "enable this option in production. Example 16-2 shows the configuration changes that\n",
      "are necessary to enable database query performance monitoring in production mode.\n",
      "238 \n",
      "| \n",
      "Chapter 16: Performance\n",
      "\n",
      "Example 16-2. config.py: configuration for slow query reporting\n",
      "class Config:\n",
      "    # ...\n",
      "    SQLALCHEMY_RECORD_QUERIES = True\n",
      "    FLASKY_SLOW_DB_QUERY_TIME = 0.5\n",
      "    # ...\n",
      "SQLALCHEMY_RECORD_QUERIES tells Flask-SQLAlchemy to enable the recording of\n",
      "query statistics. The slow query threshold is set to half a second. Both configuration\n",
      "variables were included in the base Config class, so they will be enabled for all config‐\n",
      "urations.\n",
      "Whenever a slow query is detected, an entry will be written to Flask’s application log‐\n",
      "ger. To be able to store these log entries, the logger must be configured. The logging\n",
      "configuration largely depends on the platform that hosts the application. Some exam‐\n",
      "ples are shown in Chapter 17.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 16a to check out this version of the applica‐\n",
      "tion.\n",
      "Source Code Profiling\n",
      "Another possible source of performance problems is high CPU consumption, caused\n",
      "by functions that perform heavy computing. Source code profilers are useful in find‐\n",
      "ing the slowest parts of an application. A profiler watches a running application and\n",
      "records the functions that are called and how long each takes to run. It then produces\n",
      "a detailed report showing the slowest functions.\n",
      "Profiling is typically done only in a development environment. A\n",
      "source code profiler makes the application run much slower than\n",
      "normal, because it has to observe and take notes on all that is hap‐\n",
      "pening in real time. Profiling on a production system is not recom‐\n",
      "mended, unless a lightweight profiler specifically designed to run in\n",
      "a production environment is used.\n",
      "Flask’s development web server, which comes from Werkzeug, can optionally enable\n",
      "the Python profiler for each request. Example 16-3 adds a new command-line option\n",
      "to the application that starts the web server under the profiler.\n",
      "Source Code Profiling \n",
      "| \n",
      "239\n",
      "\n",
      "Example 16-3. flasky.py: running the application under the request profiler\n",
      "@app.cli.command()\n",
      "@click.option('--length', default=25,\n",
      "              help='Number of functions to include in the profiler report.')\n",
      "@click.option('--profile-dir', default=None,\n",
      "              help='Directory where profiler data files are saved.')\n",
      "def profile(length, profile_dir):\n",
      "    \"\"\"Start the application under the code profiler.\"\"\"\n",
      "    from werkzeug.contrib.profiler import ProfilerMiddleware\n",
      "    app.wsgi_app = ProfilerMiddleware(app.wsgi_app, restrictions=[length],\n",
      "                                      profile_dir=profile_dir)\n",
      "    app.run(debug=False)\n",
      "This command attaches the ProfilerMiddleware from Werkzeug to the application,\n",
      "through its wsgi_app attribute. WSGI middlewares are invoked each time the web\n",
      "server dispatches a request to the application and can modify the way the request is\n",
      "handled, in this case by capturing profiling data. Note that the application is then\n",
      "started programmatically using the app.run() method.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 16b to check out this version of the applica‐\n",
      "tion.\n",
      "When the application is started with flask profile, the console will show the pro‐\n",
      "filer statistics for each request, which will include the slowest 25 functions. The\n",
      "--length option can be used to change the number of functions shown in the report.\n",
      "If the --profile-dir option is given, the profile data for each request is saved to a file\n",
      "in the given directory. The profiler data files can be used to generate more detailed\n",
      "reports that include a call graph. For more information on the Python profiler, con‐\n",
      "sult the official documentation.\n",
      "The preparations for deployment are complete. The next chapter will give you an\n",
      "overview of what to expect when deploying your application.\n",
      "240 \n",
      "| \n",
      "Chapter 16: Performance\n",
      "\n",
      "CHAPTER 17\n",
      "Deployment\n",
      "The web development server that comes bundled with Flask is not robust, secure, or\n",
      "efficient enough to work in a production environment. In this chapter, production\n",
      "deployment options for Flask applications are examined.\n",
      "Deployment Workflow\n",
      "Regardless of the hosting method used, there are a series of tasks that must be carried\n",
      "out when the application is installed on a production server. These include the cre‐\n",
      "ation or update of the database tables.\n",
      "Having to run these tasks manually each time the application is installed or upgraded\n",
      "is error prone and time consuming. Instead, a command that performs all the\n",
      "required tasks can be added to flasky.py.\n",
      "Example 17-1 shows a deploy command implementation that is appropriate for\n",
      "Flasky.\n",
      "Example 17-1. flasky.py: deploy command\n",
      "from flask_migrate import upgrade\n",
      "from app.models import Role, User\n",
      "@manager.command\n",
      "def deploy():\n",
      "    \"\"\"Run deployment tasks.\"\"\"\n",
      "    # migrate database to latest revision\n",
      "    upgrade()\n",
      "    # create or update user roles\n",
      "    Role.insert_roles()\n",
      "241\n",
      "\n",
      "    # ensure all users are following themselves\n",
      "    User.add_self_follows()\n",
      "The functions invoked by this command were all created before; they are just invoked\n",
      "all together from a single command to simplify the deployment of the application.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 17a to check out this version of the applica‐\n",
      "tion.\n",
      "These functions are all designed in a way that causes no problems if they are executed\n",
      "multiple times. Designing update functions in this way makes it possible to run just\n",
      "this deploy command every time an installation or upgrade is done without having to\n",
      "worry about side effects caused by a function that runs at the wrong time.\n",
      "Logging of Errors During Production\n",
      "When the application is running in debug mode, Werkzeug’s interactive debugger\n",
      "appears whenever an error occurs. The stack trace of the error is displayed on the web\n",
      "page, and it is possible to look at the source code and even evaluate expressions in the\n",
      "context of each stack frame using Flask’s interactive web-based debugger.\n",
      "The debugger is an excellent tool to debug application problems during development,\n",
      "but obviously it cannot be used in a production deployment. Errors that occur in pro‐\n",
      "duction are silenced and instead the user receives a discrete code 500 error page. But\n",
      "luckily, the stack traces of these errors are not completely lost, as Flask writes them to\n",
      "a log file.\n",
      "During startup, Flask creates an instance of Python’s logging.Logger class and\n",
      "attaches it to the application instance as app.logger. In debug mode, this logger\n",
      "writes to the console, but in production mode there are no handlers configured for it\n",
      "by default. Unless a handler is added, logs are not stored. The changes in\n",
      "Example 17-2 configure a logging handler that sends the errors that occur while run‐\n",
      "ning under the production configuration to the administrator email address config‐\n",
      "ured in the FLASKY_ADMIN setting.\n",
      "Example 17-2. config.py: sending email for application errors\n",
      "class ProductionConfig(Config):\n",
      "    # ...\n",
      "    @classmethod\n",
      "    def init_app(cls, app):\n",
      "        Config.init_app(app)\n",
      "242 \n",
      "| \n",
      "Chapter 17: Deployment\n",
      "\n",
      "        # email errors to the administrators\n",
      "        import logging\n",
      "        from logging.handlers import SMTPHandler\n",
      "        credentials = None\n",
      "        secure = None\n",
      "        if getattr(cls, 'MAIL_USERNAME', None) is not None:\n",
      "            credentials = (cls.MAIL_USERNAME, cls.MAIL_PASSWORD)\n",
      "            if getattr(cls, 'MAIL_USE_TLS', None):\n",
      "                secure = ()\n",
      "        mail_handler = SMTPHandler(\n",
      "            mailhost=(cls.MAIL_SERVER, cls.MAIL_PORT),\n",
      "            fromaddr=cls.FLASKY_MAIL_SENDER,\n",
      "            toaddrs=[cls.FLASKY_ADMIN],\n",
      "            subject=cls.FLASKY_MAIL_SUBJECT_PREFIX + ' Application Error',\n",
      "            credentials=credentials,\n",
      "            secure=secure)\n",
      "        mail_handler.setLevel(logging.ERROR)\n",
      "        app.logger.addHandler(mail_handler)\n",
      "Recall that all configuration classes have an init_app() static method that is invoked\n",
      "by create_app(), which so far has not been used. In the implementation of this\n",
      "method for the ProductionConfig class, the application logger is now configured\n",
      "with a log handler that sends errors to an email recipient.\n",
      "The logging level of the email logger is set to logging.ERROR, so only severe problems\n",
      "are going to be emailed. Messages logged on lesser levels can be logged to a file,\n",
      "syslog, or any other supported destination by adding the proper logging handlers.\n",
      "The logging method to use for these messages largely depends on the hosting plat‐\n",
      "form.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 17b to check out this version of the applica‐\n",
      "tion.\n",
      "Cloud Deployment\n",
      "The trend in application hosting is to host “in the cloud,” but this can mean many\n",
      "different things. At the most basic level, cloud hosting can mean that the application\n",
      "is installed on one or more virtual servers, which for all intents and purposes operate\n",
      "and feel like physical machines, but in reality are virtual machines managed by the\n",
      "cloud operator. An example of these types of servers are those available through the\n",
      "EC2 service from Amazon Web Services (AWS). Deploying an application to a virtual\n",
      "server is similar to doing a traditional deployment to a dedicated server, as described\n",
      "later in this chapter.\n",
      "Cloud Deployment \n",
      "| \n",
      "243\n",
      "\n",
      "A more advanced deployment model is based on containers. A container isolates an\n",
      "application in an image of the application and its environment. A container image\n",
      "includes the application plus all the dependencies it needs to run. A container plat‐\n",
      "form, such as Docker, can then install and execute a pregenerated container image on\n",
      "any system in which it runs.\n",
      "Another deployment option, formally known as Platform as a Service (PaaS), frees\n",
      "the application developer from the mundane tasks of installing and maintaining the\n",
      "hardware and software platforms on which the application runs. In the PaaS model, a\n",
      "service provider offers a fully managed platform on which applications can run. All\n",
      "the application developer needs to do is upload the application code to the servers\n",
      "maintained by the provider, after which it automatically becomes available, usually\n",
      "within seconds. Most PaaS providers offer ways to dynamically “scale” the application\n",
      "by adding or removing servers as necessary to keep up with the number of requests\n",
      "received.\n",
      "The remainder of this chapter offers an introduction to Heroku (one of the most pop‐\n",
      "ular PaaS providers), Docker containers, and finally traditional deployments, which\n",
      "are suitable for dedicated or virtual servers.\n",
      "The Heroku Platform\n",
      "Heroku was one of the first PaaS providers, having been in business since 2007. The\n",
      "Heroku platform is very flexible and supports a long list of programming languages,\n",
      "including Python. To deploy an application to Heroku, the developer uses Git to push\n",
      "the application to Heroku’s special Git server, which automatically triggers the instal‐\n",
      "lation, upgrade, configuration, and deployment of the application.\n",
      "Heroku uses units of computing called dynos to measure usage and charge for the\n",
      "service. The most common type of dyno is the web dyno, which represents a web\n",
      "server instance. An application can increase its request handling capacity by deploy‐\n",
      "ing more web dynos, each running an instance of the application. Another type of\n",
      "dyno is the worker dyno, which is used to perform background jobs or other support\n",
      "tasks.\n",
      "The platform provides a large number of plug-ins and add-ons for databases, email\n",
      "support, and many other services. The following sections expand on some of the\n",
      "details involved in deploying Flasky to Heroku.\n",
      "Preparing the Application\n",
      "To work with Heroku, the application must be hosted in a Git repository. If you are\n",
      "working with an application that is hosted on a remote Git server, such as GitHub or\n",
      "Bitbucket, cloning the application will create a local Git repository that is perfect to\n",
      "244 \n",
      "| \n",
      "Chapter 17: Deployment\n",
      "\n",
      "use with Heroku. If the application isn’t already hosted in a Git repository, you’ll need\n",
      "to create one for it on your development machine.\n",
      "If you plan on hosting your application on Heroku, it is a good idea\n",
      "to start using Git from the very beginning. GitHub has installation\n",
      "and setup guides for the three major operating systems in its help\n",
      "guide.\n",
      "Creating a Heroku account\n",
      "You must create an account with Heroku before you can use the service. Heroku pro‐\n",
      "vides a free tier that allows you to host a few simple applications, so this is a great\n",
      "platform to experiment with.\n",
      "Installing the Heroku CLI\n",
      "To work with the Heroku service, the Heroku CLI must be installed. This is a\n",
      "command-line client that manages the interactions with the service. Heroku provides\n",
      "installers for the three major operating systems.\n",
      "The first thing to do after installing the CLI is to authenticate with your Heroku\n",
      "account through the heroku login command:\n",
      "$ heroku login\n",
      "Enter your Heroku credentials.\n",
      "Email: <your-email-address>\n",
      "Password: <your-password>\n",
      "It is important that your SSH public key is uploaded to Heroku, as\n",
      "this is what enables the git push command. Normally the login\n",
      "command creates and uploads an SSH public key automatically,\n",
      "but the heroku keys:add command can be used to upload your\n",
      "public key separately from the login command or if you need to\n",
      "upload additional keys.\n",
      "Creating an application\n",
      "The next step is to create an application. Before this is done, the application needs to\n",
      "be under Git source control. If you have been using the GitHub repository to follow\n",
      "along with the code in this book, then you already have a Git repository. If not, you\n",
      "will need to create one now. To register the application with Heroku, run the follow‐\n",
      "ing command from the application’s top-level directory:\n",
      "$ heroku create <appname>\n",
      "Creating <appname>... done\n",
      "https://<appname>.herokuapp.com/ | https://git.heroku.com/<appname>.git\n",
      "The Heroku Platform \n",
      "| \n",
      "245\n",
      "\n",
      "Heroku application names must be unique across all customers, so you need to think\n",
      "of a name that is not taken by any other application. As indicated by the output of the\n",
      "create command, once deployed the application will be available at https://<app‐\n",
      "name>.herokuapp.com. Heroku also supports using a custom domain name for your\n",
      "application.\n",
      "As part of the application creation, Heroku creates a Git server dedicated to your\n",
      "application at https://git.heroku.com/<appname>.git. The create command adds this\n",
      "server to your local Git repository as a git remote with the name heroku:\n",
      "$ git remote show heroku\n",
      "* remote heroku\n",
      "  Fetch URL: https://git.heroku.com/<appname>.git\n",
      "  Push  URL: https://git.heroku.com/<appname>.git\n",
      "  HEAD branch: (unknown)\n",
      "The flask command requires the FLASK_APP environment variable to be set to work.\n",
      "To make sure that any commands that are executed in the Heroku environment suc‐\n",
      "ceed, it is a good idea to register this environment variable so that it is always set\n",
      "when Heroku executes commands related to this application. This can be done with\n",
      "the config command:\n",
      "$ heroku config:set FLASK_APP=flasky.py\n",
      "Setting FLASK_APP and restarting <appname>... done, v4\n",
      "FLASK_APP: flasky.py\n",
      "Provisioning a database\n",
      "Heroku supports Postgres databases as an add-on. The free service tier includes a\n",
      "small database of up to 10,000 rows. To attach a Postgres database to your application,\n",
      "use the following command:\n",
      "$ heroku addons:create heroku-postgresql:hobby-dev\n",
      "Creating heroku-postgresql:hobby-dev on <appname>... free\n",
      "Database has been created and is available\n",
      " ! This database is empty. If upgrading, you can transfer\n",
      " ! data from another database with pg:copy\n",
      "Created postgresql-cubic-41298 as DATABASE_URL\n",
      "Use heroku addons:docs heroku-postgresql to view documentation\n",
      "As indicated by the output of the command, once the application runs inside the Her‐\n",
      "oku platform, it will see the database location and credentials in the DATABASE_URL\n",
      "environment variable. The format of this variable is a URL, exactly in the format\n",
      "SQLAlchemy expects. Recall that the config.py script uses the value of DATABASE_URL\n",
      "if it is defined, so the connection to the Postgres database will work automatically.\n",
      "246 \n",
      "| \n",
      "Chapter 17: Deployment\n",
      "\n",
      "Configuring logging\n",
      "Logging of fatal errors by email was added earlier, but in addition to that it is impor‐\n",
      "tant to configure logging of lesser message categories. A good example of these types\n",
      "of messages are the warnings for slow database queries added in Chapter 16.\n",
      "Heroku considers any output written by the application to stdout or stderr logs, so\n",
      "a logging handler needs to be added to generate this output. The logging output is\n",
      "captured by Heroku and made accessible through the Heroku client with the heroku\n",
      "logs command.\n",
      "The logging configuration can be added to the ProductionConfig class in its\n",
      "init_app() static method. But since this type of logging is specific to Heroku, a bet‐\n",
      "ter approach is to define a new configuration specifically for this platform, leaving\n",
      "ProductionConfig as a baseline configuration for different types of production plat‐\n",
      "forms. The HerokuConfig class is shown in Example 17-3.\n",
      "Example 17-3. config.py: Heroku configuration\n",
      "class HerokuConfig(ProductionConfig):\n",
      "    @classmethod\n",
      "    def init_app(cls, app):\n",
      "        ProductionConfig.init_app(app)\n",
      "        # log to stderr\n",
      "        import logging\n",
      "        from logging import StreamHandler\n",
      "        file_handler = StreamHandler()\n",
      "        file_handler.setLevel(logging.INFO)\n",
      "        app.logger.addHandler(file_handler)\n",
      "When the application is executed by Heroku, it needs to know that this new configu‐\n",
      "ration needs to be used. The application instance created in flasky.py uses the\n",
      "FLASK_CONFIG environment variable to know what configuration to use, so this vari‐\n",
      "able needs to be set appropriately in the Heroku environment. Environment variables\n",
      "for the Heroku environment are set using the Heroku client’s config:set command:\n",
      "$ heroku config:set FLASK_CONFIG=heroku\n",
      "Setting FLASK_CONFIG and restarting <appname>... done, v4\n",
      "FLASK_CONFIG: heroku\n",
      "To increase the security of your application, it is a good idea to configure a difficult-\n",
      "to-guess string as the application’s secret key, which is used to sign the user session\n",
      "and the authentication tokens. The Config base class includes the SECRET_KEY\n",
      "attribute for this purpose, and sets its value from an environment variable of the same\n",
      "name if it exists. When working on the application in your development system it is\n",
      "okay to leave this variable undefined and let the Config class configure a hardcoded\n",
      "The Heroku Platform \n",
      "| \n",
      "247\n",
      "\n",
      "value, but on a production platform it is extremely important to set a strong secret\n",
      "key that is not known to anyone, since a leaked key will enable an attacker to forge\n",
      "the contents of the user session or generate valid tokens. To make your key secure,\n",
      "just set the SECRET_KEY environment variable to a unique string that is not stored\n",
      "anywhere:\n",
      "$ heroku config:set SECRET_KEY=d68653675379485599f7876a3b469a57\n",
      "Setting SECRET_KEY and restarting <appname>... done, v4\n",
      "SECRET_KEY: d68653675379485599f7876a3b469a57\n",
      "There are many ways to generate random strings that are appropriate to be used as\n",
      "secret keys. You can do so with Python as follows:\n",
      "(venv) $ python -c \"import uuid; print(uuid.uuid4().hex)\"\n",
      "d68653675379485599f7876a3b469a57\n",
      "Configuring email\n",
      "Heroku does not provide an SMTP server, so an external server must be configured.\n",
      "There are several third-party add-ons that integrate production-ready email sending\n",
      "support with Heroku, but for testing and evaluation purposes it is sufficient to use the\n",
      "default Gmail configuration inherited from the base Config class.\n",
      "Because it can be a security risk to embed login credentials directly in the script, the\n",
      "username and password to access the Gmail SMTP server are provided as environ‐\n",
      "ment variables (if you haven’t yet, it is a very good idea that instead of using your per‐\n",
      "sonal email account you create a secondary email to use for testing):\n",
      "$ heroku config:set MAIL_USERNAME=<your-gmail-username>\n",
      "$ heroku config:set MAIL_PASSWORD=<your-gmail-password>\n",
      "Adding a top-level requirements file\n",
      "Heroku installs package dependencies from a requirements.txt file stored in the top-\n",
      "level directory of the application. All the dependencies in this file will be imported\n",
      "into a virtual environment managed by Heroku as part of the deployment.\n",
      "The Heroku requirements file must include all the common requirements for the\n",
      "production version of the application, plus the psycopg2 package that enables SQL‐\n",
      "Alchemy to access the Postgres database. A heroku.txt file with these dependencies\n",
      "can be added in the requirements directory and then imported from the top-level\n",
      "requirements.txt file as shown in Example 17-4.\n",
      "Example 17-4. requirements.txt: Heroku requirements file\n",
      "-r requirements/heroku.txt\n",
      "248 \n",
      "| \n",
      "Chapter 17: Deployment\n",
      "\n",
      "Enabling Secure HTTP with Flask-SSLify\n",
      "When the user logs in to the application by submitting a username and a password in\n",
      "a web form, these values are at risk of being intercepted by a malicious third party, as\n",
      "discussed several times before. During development this is not a problem, but this\n",
      "risk needs to be eliminated when you deploy the application on a production server.\n",
      "To prevent user credentials from being exposed while in transit, it is necessary to use\n",
      "secure HTTP, which encrypts all the communications between clients and the server\n",
      "using public key cryptography.\n",
      "Heroku makes all applications that are accessed on the herokuapp.com domain avail‐\n",
      "able on both http:// and https:// without any configuration required. Because the\n",
      "application runs on Heroku’s domain, it will use Heroku’s own SSL certificate. The\n",
      "only necessary action to fully secure the application is to intercept any requests sent\n",
      "to the http:// interface and redirect them to https://, which is exactly what the Flask-\n",
      "SSLify extension does.\n",
      "As usual, Flask-SSLify is installed with pip:\n",
      "(venv) $ pip install flask-sslify\n",
      "The code that activates this extension is added to the application factory function, as\n",
      "shown in Example 17-5.\n",
      "Example 17-5. app/__init__.py: redirecting all requests to secure HTTP\n",
      "def create_app(config_name):\n",
      "    # ...\n",
      "    if app.config['SSL_REDIRECT']:\n",
      "        from flask_sslify import SSLify\n",
      "        sslify = SSLify(app)\n",
      "    # ...\n",
      "Support for SSL needs to be enabled only in production mode, and only when the\n",
      "platform supports it. To make it easy to switch SSL on and off, a new configuration\n",
      "variable called SSL_REDIRECT is added. The base Config class sets it to False, so that\n",
      "SSL redirects are not used by default, and the class HerokuConfig overrides it so that\n",
      "only on that configuration are the redirects issued. The implementation of this con‐\n",
      "figuration variable is shown in Example 17-6.\n",
      "The Heroku Platform \n",
      "| \n",
      "249\n",
      "\n",
      "Example 17-6. config.py: configuring the use of SSL\n",
      "class Config:\n",
      "    # ...\n",
      "    SSL_REDIRECT = False\n",
      "class HerokuConfig(ProductionConfig):\n",
      "    # ...\n",
      "    SSL_REDIRECT = True if os.environ.get('DYNO') else False\n",
      "The value of SSL_REDIRECT in HerokuConfig is only set to True if the environment\n",
      "variable DYNO exists. This variable is set by Heroku in its environment, so using the\n",
      "Heroku configuration for local testing does not activate the SSL redirects.\n",
      "With these changes, the users will be forced to use the SSL server when accessing the\n",
      "application on Heroku—but there is one more detail that needs to be handled to\n",
      "make this support complete. When using Heroku, clients do not connect to the appli‐\n",
      "cation directly but to a reverse proxy server. The reverse proxy server receives requests\n",
      "from many applications, and forwards them to each of them as appropriate. In this\n",
      "type of setup, only the proxy server runs in SSL mode; the SSL connection is termi‐\n",
      "nated at the proxy server, and applications receive the forwarded requests from the\n",
      "proxy server without encryption. This presents a problem when the application needs\n",
      "to generate absolute URLs, because in the Flask application the request object\n",
      "describes the forwarded request, which is not encrypted, and not the original request\n",
      "sent by the client through an encrypted connection.\n",
      "An example of the problem this can cause is with the generation of account confirma‐\n",
      "tion or password reset links that are sent by email to users. When url_for() is called\n",
      "with _external=True to generate an absolute URL for these links, Flask will use\n",
      "http:// for them, because it does not know that there is a reverse proxy that is accept‐\n",
      "ing encrypted connections from the outside.\n",
      "Proxy servers pass information that describes the original request from the client to\n",
      "the redirected web servers through custom HTTP headers, so it is possible to deter‐\n",
      "mine whether the user is communicating with the application over SSL by looking at\n",
      "these headers. Werkzeug provides a WSGI middleware that checks the custom head‐\n",
      "ers from the proxy server and updates the request object accordingly so that, for\n",
      "example, request.is_secure reflects the encryption state of the request that the cli‐\n",
      "ent sent to the reverse proxy server and not the request that the proxy server then\n",
      "forwarded to the application. Example 17-7 shows how to add the ProxyFix middle‐\n",
      "ware to the application.\n",
      "250 \n",
      "| \n",
      "Chapter 17: Deployment\n",
      "\n",
      "Example 17-7. config.py: adding support for proxy servers\n",
      "class HerokuConfig(ProductionConfig):\n",
      "    # ...\n",
      "    @classmethod\n",
      "    def init_app(cls, app):\n",
      "        # ...\n",
      "        # handle reverse proxy server headers\n",
      "        from werkzeug.contrib.fixers import ProxyFix\n",
      "        app.wsgi_app = ProxyFix(app.wsgi_app)\n",
      "The middleware is added in the initialization method for the Heroku configuration.\n",
      "WSGI middlewares such as ProxyFix are added by wrapping the WSGI application.\n",
      "When a request comes, the middlewares get a chance to inspect the environment and\n",
      "make changes before the request is processed. The ProxyFix middleware is necessary\n",
      "not only for Heroku but in any deployment that uses a reverse proxy server.\n",
      "Running a production web server\n",
      "Heroku expects applications to start their own production web server and configure\n",
      "it to listen to requests on the port number set in the environment variable PORT.\n",
      "The development web server that comes with Flask will perform very poorly in this\n",
      "situation because it is not designed to run in a production environment. Two \n",
      "production-ready web servers that work well with Flask applications are Gunicorn\n",
      "and uWSGI.\n",
      "It is a good idea to install the chosen web server in the local virtual environment, so\n",
      "that it can be tested in a way similar to how it will run in the Heroku environment.\n",
      "For example, Gunicorn is installed as follows:\n",
      "(venv) $ pip install gunicorn\n",
      "To run the application locally under Gunicorn, use the following command:\n",
      "(venv) $ gunicorn flasky:app\n",
      "[2017-08-03 23:54:36 -0700] [INFO] Starting gunicorn 19.7.1\n",
      "[2017-08-03 23:54:36 -0700] [INFO] Listening at: http://127.0.0.1:8000 (68982)\n",
      "[2017-08-03 23:54:36 -0700] [INFO] Using worker: sync\n",
      "[2017-08-03 23:54:36 -0700] [INFO] Booting worker with pid: 68985\n",
      "The flasky:app argument tells Gunicorn where the application instance is located.\n",
      "The name given before the colon is the package or module that defines this instance,\n",
      "while the name after the colon is the actual application instance name. Note that\n",
      "Gunicorn uses port 8000 by default, not 5000 like Flask. Like the Flask development\n",
      "web server, you can exit Gunicorn with Ctrl+C.\n",
      "The Heroku Platform \n",
      "| \n",
      "251\n",
      "\n",
      "The Gunicorn web server does not work on Microsoft Windows.\n",
      "The other recommended web server, uWSGI, does work on Win‐\n",
      "dows, but it can be difficult to install due to it being written in\n",
      "native code. If you want to test the Heroku deployment on your\n",
      "Windows system, you can use Waitress, which is another pure\n",
      "Python web server that is in many ways similar to Gunicorn but\n",
      "has the advantage that it fully supports Windows. Waitress is\n",
      "installed with pip:\n",
      "(venv) $ pip install waitress\n",
      "To start the Waitress web server, use the waitress-serve com‐\n",
      "mand:\n",
      "(venv) $ waitress-serve --port 8000 flasky:app\n",
      "Adding a Procfile\n",
      "Heroku needs to know what command to use to start the application. This command\n",
      "is given in a special file called Procfile. This file must be included in the top-level\n",
      "directory of the application.\n",
      "Example 17-8 shows the contents of this file.\n",
      "Example 17-8. Procfile: Heroku Procfile\n",
      "web: gunicorn flasky:app\n",
      "The format for the Procfile is very simple: in each line a task name is given, followed\n",
      "by a colon and then the command that runs the task. The task name web is special; it\n",
      "is recognized by Heroku as the task that starts the web server. Heroku will give this\n",
      "task a PORT environment variable set to the port on which the application needs to\n",
      "listen for requests. Gunicorn by default honors the PORT variable if it is set in the\n",
      "environment, so there is no need to include it in the startup command.\n",
      "If you are using Microsoft Windows, or need your application to be\n",
      "fully compatible with that platform, you can instead use the Wait‐\n",
      "ress web server:\n",
      "web: waitress-serve --port=$PORT flasky:app\n",
      "Applications can declare additional tasks with names other than\n",
      "web in the Procfile. Each task included in the Procfile will be started\n",
      "on its own dyno.\n",
      "252 \n",
      "| \n",
      "Chapter 17: Deployment\n",
      "\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 17c to check out this version of the applica‐\n",
      "tion. If you are using Microsoft Windows, run git checkout 17c-\n",
      "waitress to check out a version of the application configured to\n",
      "use the Waitress web server instead of Gunicorn.\n",
      "Testing with Heroku Local\n",
      "The Heroku CLI includes the local command, used to run the application locally in\n",
      "a very similar way to how it runs on the Heroku servers. However, environment vari‐\n",
      "ables such as FLASK_APP are not available when running the application locally. The\n",
      "heroku local command looks for environment variables that configure the applica‐\n",
      "tion in a file named .env in the top-level directory of the application. For example,\n",
      "the .env file can contain the following variables:\n",
      "FLASK_APP=flasky.py\n",
      "FLASK_CONFIG=heroku\n",
      "MAIL_USERNAME=<your-gmail-username>\n",
      "MAIL_PASSWORD=<your-gmail-password>\n",
      "Because the .env file contains passwords and other sensitive\n",
      "account information, it should never be added to source control.\n",
      "Before the application can be started, the deployment task needs to be executed to set\n",
      "up the database. One-off tasks can be executed with the local:run command:\n",
      "(venv) $ heroku local:run flask deploy\n",
      "[OKAY] Loaded ENV .env File as KEY=VALUE Format\n",
      "INFO  Context impl SQLiteImpl.\n",
      "INFO  Will assume non-transactional DDL.\n",
      "INFO  Running upgrade  -> 38c4e85512a9, initial migration\n",
      "INFO  Running upgrade 38c4e85512a9 -> 456a945560f6, login support\n",
      "INFO  Running upgrade 456a945560f6 -> 190163627111, account confirmation\n",
      "INFO  Running upgrade 190163627111 -> 56ed7d33de8d, user roles\n",
      "INFO  Running upgrade 56ed7d33de8d -> d66f086b258, user information\n",
      "INFO  Running upgrade d66f086b258 -> 198b0eebcf9, caching of avatar hashes\n",
      "INFO  Running upgrade 198b0eebcf9 -> 1b966e7f4b9e, post model\n",
      "INFO  Running upgrade 1b966e7f4b9e -> 288cd3dc5a8, rich text posts\n",
      "INFO  Running upgrade 288cd3dc5a8 -> 2356a38169ea, followers\n",
      "INFO  Running upgrade 2356a38169ea -> 51f5ccfba190, comments\n",
      "The Heroku Platform \n",
      "| \n",
      "253\n",
      "\n",
      "The heroku local command reads the Procfile and executes the tasks defined by it:\n",
      "(venv) $ heroku local\n",
      "[OKAY] Loaded ENV .env File as KEY=VALUE Format\n",
      "11:37:49 AM web.1 |  [INFO] Starting gunicorn 19.7.1\n",
      "11:37:49 AM web.1 |  [INFO] Listening at: http://0.0.0.0:5000 (91686)\n",
      "11:37:49 AM web.1 |  [INFO] Using worker: sync\n",
      "11:37:49 AM web.1 |  [INFO] Booting worker with pid: 91689\n",
      "The logging output of all the tasks started by this command is consolidated into a sin‐\n",
      "gle stream that is printed to the console, with each line prefixed with a timestamp and\n",
      "the task name.\n",
      "The heroku local command also allows simulation of the use of multiple dynos to\n",
      "scale the application. The following command starts three web workers, each listen‐\n",
      "ing on a different port:\n",
      "(venv) $ heroku local web=3\n",
      "Deploying with git push\n",
      "The final step in the process is to upload the application to the Heroku servers. Make\n",
      "sure that all the changes are committed to the local Git repository and then use git\n",
      "push heroku master to upload the application to the heroku remote:\n",
      "$ git push heroku master\n",
      "Counting objects: 502, done.\n",
      "Delta compression using up to 8 threads.\n",
      "Compressing objects: 100% (426/426), done.\n",
      "Writing objects: 100% (502/502), 108.03 KiB | 0 bytes/s, done.\n",
      "Total 502 (delta 303), reused 146 (delta 61)\n",
      "remote: Compressing source files... done.\n",
      "remote: Building source:\n",
      "remote:\n",
      "remote: -----> Python app detected\n",
      "remote: -----> Installing python-3.6.2\n",
      "remote: -----> Installing pip\n",
      "remote: -----> Installing requirements with pip\n",
      "...\n",
      "remote: -----> Discovering process types\n",
      "remote:        Procfile declares types -> web\n",
      "remote:\n",
      "remote: -----> Compressing...\n",
      "remote:        Done: 49.4M\n",
      "remote: -----> Launching...\n",
      "remote:        Released v8\n",
      "remote:        https://<appname>.herokuapp.com/ deployed to Heroku\n",
      "remote:\n",
      "remote: Verifying deploy... done.\n",
      "To https://git.heroku.com/<appname>.git\n",
      " * [new branch]      master -> master\n",
      "254 \n",
      "| \n",
      "Chapter 17: Deployment\n",
      "\n",
      "The application is now deployed and running, but it is not going to work correctly\n",
      "because the deploy command that initializes the database tables has not been exe‐\n",
      "cuted yet. The Heroku client can run this command as follows:\n",
      "$ heroku run flask deploy\n",
      "Running flask deploy on <appname>... up, run.3771 (Free)\n",
      "INFO  [alembic.runtime.migration] Context impl PostgresqlImpl.\n",
      "INFO  [alembic.runtime.migration] Will assume transactional DDL.\n",
      "...\n",
      "After the database tables are created and configured, the application can be restarted\n",
      "so that it starts cleanly with an updated database:\n",
      "$ heroku restart\n",
      "Restarting dynos on <appname>... done\n",
      "The application should now be fully deployed and online at https://<appname>.hero‐\n",
      "kuapp.com.\n",
      "Reviewing application logs\n",
      "The logging output generated by the application is captured by Heroku. To view the\n",
      "contents of the log, use the logs command:\n",
      "$ heroku logs\n",
      "During testing it can also be convenient to “tail” the log file, which can be done as\n",
      "follows:\n",
      "$ heroku logs -t\n",
      "Deploying an Upgrade\n",
      "When a Heroku application needs to be upgraded the same process needs to be\n",
      "repeated. After all the changes have been committed to the Git repository, the follow‐\n",
      "ing commands perform an upgrade:\n",
      "$ heroku maintenance:on\n",
      "$ git push heroku master\n",
      "$ heroku run flask deploy\n",
      "$ heroku restart\n",
      "$ heroku maintenance:off\n",
      "The maintenance option available on the Heroku CLI will take the application offline\n",
      "during the upgrade and will show a static page that informs users that the site will be\n",
      "coming back soon. This prevents users from accessing the application while it is\n",
      "going through the upgrade process.\n",
      "The Heroku Platform \n",
      "| \n",
      "255\n",
      "\n",
      "Docker Containers\n",
      "You are now familiar with Heroku, which is a fairly high-level deployment option. In\n",
      "this section you will learn how to work with containers, and in particular with the\n",
      "Docker platform, which is not as automated as a PaaS but provides more flexibility\n",
      "and is not tied to a specific cloud provider.\n",
      "Containers are a special type of virtual machine that run on top of the kernel of the\n",
      "host operating system, unlike standard virtual machines, which have their own vir‐\n",
      "tualized kernel and hardware. Because the virtualization stops at the kernel, contain‐\n",
      "ers are much more lightweight and efficient than virtual machines, but they require\n",
      "dedicated support built into the operating system. The Linux kernel has full support\n",
      "for containers.\n",
      "Installing Docker\n",
      "The most popular container platform is Docker, which has a free Community Edition\n",
      "(known as Docker CE) and a subscription-based Enterprise Edition (Docker EE).\n",
      "Docker can be installed on the three major desktop operating systems, and also on\n",
      "cloud servers. The easiest way to develop and test a “containerized” application is to\n",
      "install Docker CE on your development system. For macOS and Microsoft Windows\n",
      "there are one-click installers available from the Docker Store. This page also includes\n",
      "installation instructions for CentOS, Fedora, Debian, and Ubuntu Linux distribu‐\n",
      "tions.\n",
      "After you complete the installation of Docker CE on your system, you should be able\n",
      "to access the docker command from your terminal:\n",
      "$ docker version\n",
      "Client:\n",
      " Version:      17.06.0-ce\n",
      " API version:  1.30\n",
      " Go version:   go1.8.3\n",
      " Git commit:   02c1d87\n",
      " Built:        Fri Jun 23 21:31:53 2017\n",
      " OS/Arch:      darwin/amd64\n",
      "Server:\n",
      " Version:      17.06.0-ce\n",
      " API version:  1.30 (minimum version 1.12)\n",
      " Go version:   go1.8.3\n",
      " Git commit:   02c1d87\n",
      " Built:        Fri Jun 23 21:51:55 2017\n",
      " OS/Arch:      linux/amd64\n",
      " Experimental: true\n",
      "256 \n",
      "| \n",
      "Chapter 17: Deployment\n",
      "\n",
      "Docker for Windows requires Microsoft’s Hyper-V feature to be\n",
      "enabled. The installer will normally enable it for you, but if Docker\n",
      "does not appear to work correctly after installation, the state of the\n",
      "Hyper-V hypervisor is the first thing to check. You should keep in\n",
      "mind that enabling Hyper-V on your Windows machine will pre‐\n",
      "vent other hypervisors (such as Oracle’s VirtualBox) from working.\n",
      "If your system does not support Hyper-V virtualization, or you\n",
      "need a Docker solution that does not render other virtualization\n",
      "technologies unusable, you may want to install Docker Toolbox, a\n",
      "legacy Docker product for Windows that is based on VirtualBox.\n",
      "Building a Container Image\n",
      "The first task when working with containers is to build a container image for the\n",
      "application. An image is a snapshot of a container’s filesystem, used as a template\n",
      "when starting new containers. Docker expects the instructions to create the image to\n",
      "be provided in a file named Dockerfile. Example 17-9 shows a Dockerfile that builds\n",
      "the application featured in this book.\n",
      "Example 17-9. Dockerfile: container image build script\n",
      "FROM python:3.6-alpine\n",
      "ENV FLASK_APP flasky.py\n",
      "ENV FLASK_CONFIG docker\n",
      "RUN adduser -D flasky\n",
      "USER flasky\n",
      "WORKDIR /home/flasky\n",
      "COPY requirements requirements\n",
      "RUN python -m venv venv\n",
      "RUN venv/bin/pip install -r requirements/docker.txt\n",
      "COPY app app\n",
      "COPY migrations migrations\n",
      "COPY flasky.py config.py boot.sh ./\n",
      "# runtime configuration\n",
      "EXPOSE 5000\n",
      "ENTRYPOINT [\"./boot.sh\"]\n",
      "The build commands that can be included in a Dockerfile are documented in detail in\n",
      "the Dockerfile reference. In essence, these are deployment commands that install and\n",
      "configure the application in the container’s filesystem, which is isolated from your\n",
      "system.\n",
      "Docker Containers \n",
      "| \n",
      "257\n",
      "\n",
      "The FROM command is required in all Dockerfiles to specify a base container image to\n",
      "start from. In most cases, this is going to be an image that is publicly available in\n",
      "Docker Hub, Docker’s container image repository. The repository contains official\n",
      "images for several Python interpreter versions. These are images that have a base\n",
      "operating system with Python installed on it. Images are specified with a name and a\n",
      "tag. The name of the official Docker Hub Python image is simply python. The differ‐\n",
      "ent tags that are available can be seen in the Docker Hub page for the image. For the\n",
      "python image, tags are used to specify the desired interpreter version and platform.\n",
      "For this application, a 3.6 interpreter built on top of the Alpine Linux distribution is\n",
      "used. Alpine Linux is a platform commonly used in container images due to its small\n",
      "size.\n",
      "The macOS and Windows versions of Docker are able to run\n",
      "Linux-based containers.\n",
      "The ENV command defines runtime environment variables. This command takes two\n",
      "arguments: a variable name and its value. Any environment variables defined with\n",
      "this command will be available when a container based on this image is executed. The\n",
      "FLASK_APP variable required by the flask command is defined here, as is\n",
      "FLASK_CONFIG, which is the name of the configuration class the application uses to\n",
      "configure itself when it starts. The Docker deployment will use a new configuration\n",
      "called docker, implemented in a DockerConfig class as shown in Example 17-10. This\n",
      "new configuration class inherits from ProductionConfig and just configures logging\n",
      "to be directed to stderr, which Docker automatically captures and exposes through\n",
      "the docker logs command.\n",
      "Example 17-10. config.py: Docker configuration\n",
      "class DockerConfig(ProductionConfig):\n",
      "    @classmethod\n",
      "    def init_app(cls, app):\n",
      "        ProductionConfig.init_app(app)\n",
      "        # log to stderr\n",
      "        import logging\n",
      "        from logging import StreamHandler\n",
      "        file_handler = StreamHandler()\n",
      "        file_handler.setLevel(logging.INFO)\n",
      "        app.logger.addHandler(file_handler)\n",
      "config = {\n",
      "    # ...\n",
      "258 \n",
      "| \n",
      "Chapter 17: Deployment\n",
      "\n",
      "    'docker': DockerConfig,\n",
      "    # ...\n",
      "}\n",
      "The RUN command executes a command in the context of the container image. In the\n",
      "first occurrence of RUN, a flasky user is created inside the container. The adduser com‐\n",
      "mand is part of Alpine Linux, and is available in the base image selected by the FROM\n",
      "command. The -D argument to adduser suppresses an interactive prompt for the\n",
      "user’s password.\n",
      "The USER command selects the user under which the container will run, and also the\n",
      "user for the remaining commands in the Dockerfile. Docker uses the root user by\n",
      "default, but it is considered a good practice to switch to a regular user when root\n",
      "access isn’t needed.\n",
      "The WORKDIR command defines the top-level directory where the application is going\n",
      "to be installed. For this application, the home directory for the newly created flasky\n",
      "user is used. The remaining commands in the Dockerfile will execute with this direc‐\n",
      "tory as the current directory.\n",
      "The COPY command copies files from the local filesystem to the container’s filesystem.\n",
      "The requirements, app, and migrations directories are copied in their entirety, and\n",
      "then the top-level flasky.py, config.py, and new boot.sh files (discussed shortly) are\n",
      "copied as well.\n",
      "The two additional RUN commands create a virtual environment and install the\n",
      "requirements in it. A dedicated requirements file was created for Docker as require‐\n",
      "ments/docker.txt. This file imports all the dependencies from requirements/\n",
      "common.txt and adds Gunicorn, which will be used as a web server as in the Heroku\n",
      "deployment.\n",
      "The EXPOSE command defines the port on which the application running inside the\n",
      "container will install its server. When the container is started, Docker will map this\n",
      "port to a real port on the host machine, so that the container can receive requests\n",
      "from the outside world.\n",
      "The final command is ENTRYPOINT. This command specifies how to execute the appli‐\n",
      "cation when the container is started. The new boot.sh file, copied into the container\n",
      "above, is used as the startup script. Example 17-11 shows the contents of this file.\n",
      "Example 17-11. boot.sh: container startup script\n",
      "#!/bin/sh\n",
      "source venv/bin/activate\n",
      "flask deploy\n",
      "exec gunicorn -b 0.0.0.0:5000 --access-logfile - --error-logfile - flasky:app\n",
      "Docker Containers \n",
      "| \n",
      "259\n",
      "\n",
      "The script starts by activating the venv virtual environment that was created as part of\n",
      "the build. Then it runs the application’s deploy command, built earlier in this chapter\n",
      "and also used for the Heroku deployment. This will create a new database, upgrade it\n",
      "to the latest version, and insert the default roles. Because the DATABASE_URL environ‐\n",
      "ment variable hasn’t been set, the database will use the SQLite engine. Then a Guni‐\n",
      "corn server listening on port 5000 is started. Docker captures all the output from the\n",
      "application and presents it as logs, so Gunicorn is configured to write both its access\n",
      "and error log files to standard output. Starting Gunicorn with exec makes the Guni‐\n",
      "corn process take over the process running the boot.sh file. This is done because\n",
      "Docker pays special attention to the process that starts a container, and expects it to\n",
      "be the main process throughout its life. When this process ends, the container ends as\n",
      "well.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 17d to check out this version of the applica‐\n",
      "tion.\n",
      "A container image for Flasky can now be built as follows:\n",
      "$ docker build -t flasky:latest .\n",
      "Sending build context to Docker daemon  51.08MB\n",
      "Step 1/14 : FROM python:3.6-alpine\n",
      " ---> a6beab4fa70b\n",
      "...\n",
      "Successfully built 930e17a89b42\n",
      "Successfully tagged flasky:latest\n",
      "The -t argument to docker build gives a name and tag to the container image, sepa‐\n",
      "rated by a colon. The latest tag name is typically used for the most up-to-date ver‐\n",
      "sion of a container image. The dot at the end of the build command sets the current\n",
      "directory as the top-level directory during the build. Docker will look for the Docker‐\n",
      "file in this directory, and will also make the files in this directory and all sub-\n",
      "directories available to be added to the container image.\n",
      "As a result of a successful docker build command, the built container image is\n",
      "stored in a local image repository. The docker images command shows the contents\n",
      "of the image repository on your system:\n",
      "$ docker images\n",
      "REPOSITORY   TAG          IMAGE ID      CREATED         SIZE\n",
      "flasky       latest       930e17a89b42  5 minutes ago   127MB\n",
      "python       3.6-alpine   a6beab4fa70b  3 weeks ago     88.7MB\n",
      "260 \n",
      "| \n",
      "Chapter 17: Deployment\n",
      "\n",
      "This listing includes the just-built flasky:latest image and also the base Python 3.6\n",
      "interpreter image referenced in the FROM Dockerfile, which Docker downloads and\n",
      "installs as part of the build.\n",
      "Running a Container\n",
      "Once a container image for the application is built, all that remains is to run it. The\n",
      "docker run command makes this a very simple task:\n",
      "$ docker run --name flasky -d -p 8000:5000 \\\n",
      "  -e SECRET_KEY=57d40f677aff4d8d96df97223c74d217 \\\n",
      "  -e MAIL_USERNAME=<your-gmail-username> \\\n",
      "  -e MAIL_PASSWORD=<your-gmail-password> flasky:latest\n",
      "The --name option gives the container a name. Naming containers is optional; if a\n",
      "name is not given, Docker generates one using randomly selected words.\n",
      "The -d option starts the container in detached mode, which means that the container\n",
      "will run in the background on your system. A container that is not detached runs as a\n",
      "foreground task attached to the console session.\n",
      "The -p option maps port 8000 in the host system to port 5000 inside the container.\n",
      "Docker provides the flexibility of mapping container ports to any port in the host sys‐\n",
      "tem. This mapping enables two or more instances of the same container image to run\n",
      "on different host ports, while each instance uses its own virtualized port 5000.\n",
      "The -e option defines environment variables that are going to exist in the context of\n",
      "the container, in addition to any variables defined at build time with the ENV com‐\n",
      "mand in the Dockerfile. The value assigned to the SECRET_KEY variable ensures that\n",
      "user sessions and tokens are signed with a unique and very hard to guess key. You\n",
      "should generate your own unique key for this variable. The values for the\n",
      "MAIL_USERNAME and MAIL_PASSWORD variables configure email sending through the\n",
      "Gmail service. For a production deployment that uses a different email service pro‐\n",
      "vider the MAIL_SERVER, MAIL_PORT, and MAIL_USE_TLS variables should be defined as\n",
      "well.\n",
      "The final argument in the docker run command is the container image and tag to\n",
      "execute. This should match the name and tag given as the -t option to the docker\n",
      "build command.\n",
      "When the container starts in the background, the docker run command prints the\n",
      "container ID to the console. This is a 256-bit unique identifier printed in hexadecimal\n",
      "notation. This ID can be used in any commands that require a reference to a con‐\n",
      "tainer (in practice, only the first few characters of the ID need to be provided, such\n",
      "that the container can be uniquely identified).\n",
      "Docker Containers \n",
      "| \n",
      "261\n",
      "\n",
      "To confirm that the container is running, the docker ps command can be used:\n",
      "$ docker ps\n",
      "CONTAINER ID  IMAGE          CREATED     STATUS     PORTS                  NAMES\n",
      "71357ee776ae  flasky:latest  4 secs ago  Up 8 secs  0.0.0.0:8000->5000/tcp flasky\n",
      "Since the container is now up and running, you can access the containerized applica‐\n",
      "tion on port 8000 of your system, either locally as http://localhost:8000 or from any\n",
      "other computer in the network as http://<ip-address>:8000.\n",
      "To stop this container, use the docker stop command:\n",
      "$ docker stop 71357ee776ae\n",
      "71357ee776ae\n",
      "The stop command stops the container but does not remove it from the system. To\n",
      "remove it, use the docker rm command:\n",
      "$ docker rm 71357ee776ae\n",
      "71357ee776ae\n",
      "These two operations can be combined into one with docker rm -f:\n",
      "$ docker rm -f 71357ee776ae\n",
      "71357ee776ae\n",
      "Inspecting a Running Container\n",
      "When a container appears to misbehave, it might be necessary to debug it. The most\n",
      "obvious debugging mechanism is to add logging statements to the application and\n",
      "then monitor the running container with the docker logs command.\n",
      "In some situations, however, it might be more convenient to open a shell session on\n",
      "the running container so that it can be inspected more closely. The docker exec\n",
      "command makes this possible:\n",
      "$ docker exec -it 71357ee776ae sh\n",
      "In this example, Docker is going to open a shell session with sh (the Unix shell)\n",
      "without interrupting the container. The -it options connect the terminal session\n",
      "from which the command is issued to the new process, so that the shell can be oper‐\n",
      "ated interactively. If the container includes other, more advanced shells such as bash\n",
      "or even a Python interpreter, they can be used as well.\n",
      "A common strategy when troubleshooting containers is to create a special image\n",
      "loaded with additional tools such as a debugger that can later be invoked from a shell\n",
      "session.\n",
      "262 \n",
      "| \n",
      "Chapter 17: Deployment\n",
      "\n",
      "Pushing Your Container Image to an External Registry\n",
      "Having a container image locally is convenient when developing and testing an appli‐\n",
      "cation, but when you are ready to share the image with others, you have to push it to\n",
      "an external registry server.\n",
      "The Docker Hub registry is Docker’s image repository, a convenient service where\n",
      "you can host your images. A free Docker Hub account allows you to store an unlimi‐\n",
      "ted number of public container images, but only one private image. Paid plans\n",
      "increase the number of private images you can host. To create your Docker Hub\n",
      "account, go to https://hub.docker.com.\n",
      "Once you have a Docker Hub account, you can log in to it from the command line\n",
      "with the docker login command:\n",
      "$ docker login\n",
      "Login with your Docker ID to push and pull images from Docker Hub.\n",
      "Username: <your-dockerhub-username>\n",
      "Password: <your-dockerhub-password>\n",
      "Login Succeeded\n",
      "To log in to a container image repository other than Docker Hub,\n",
      "pass the address of your repository as an argument to docker\n",
      "login.\n",
      "Local container images are given a simple name. To prepare to push an image to\n",
      "Docker Hub, the image name must be prefixed with the Docker Hub account name\n",
      "and a slash as a separator. The flasky:latest image built earlier can be given a sec‐\n",
      "ondary name properly formatted for pushing to Docker Hub with the docker tag\n",
      "command:\n",
      "$ docker tag flasky:latest <your-dockerhub-username>/flasky:latest\n",
      "To upload the image to Docker Hub, use the docker push command:\n",
      "$ docker push <your-dockerhub-username>/flasky:latest\n",
      "The container image is now publicly available, and anybody can start a container\n",
      "based on it with the docker run command:\n",
      "$ docker run --name flasky -d -p 8000:5000 \\\n",
      "<your-dockerhub-username>/flasky:latest\n",
      "Docker Containers \n",
      "| \n",
      "263\n",
      "\n",
      "Using an External Database\n",
      "One disadvantage of the way Flasky was deployed as a Docker container is that the\n",
      "default SQLite database lives in the same container as the application. This makes it\n",
      "very difficult to perform an upgrade, because once a running container is stopped,\n",
      "the database is gone with it.\n",
      "A better approach is to host the database server separately from the application con‐\n",
      "tainer. That makes upgrading the application while preserving the database an easy\n",
      "task, since all that is needed is to replace the application container with a new one.\n",
      "Docker promotes a modular approach to building an application, in which each ser‐\n",
      "vice is hosted in its own container. There are public container images available for\n",
      "MySQL, Postgres, and many other database servers. The docker run command can\n",
      "be used to deploy any of these directly to your system. The following command\n",
      "deploys a MySQL 5.7 database server to your system:\n",
      "$ docker run --name mysql -d -e MYSQL_RANDOM_ROOT_PASSWORD=yes \\\n",
      "  -e MYSQL_DATABASE=flasky -e MYSQL_USER=flasky \\\n",
      "  -e MYSQL_PASSWORD=<database-password> \\\n",
      "  mysql/mysql-server:5.7 \n",
      "This command creates a container named mysql that runs in the background. The -e\n",
      "option assigns a few environment variables that this container takes as configuration.\n",
      "These and many other variables are documented in the Docker Hub page for the\n",
      "MySQL image. The preceding command configures the database with a randomly\n",
      "generated root password (use docker logs mysql right after starting the container to\n",
      "see the assigned password in the logs), and with a brand-new database called flasky\n",
      "that is configured to be accessed by a user named flasky as well. You need to provide a\n",
      "secure password for this user as a value for the MYSQL_PASSWORD environment vari‐\n",
      "able.\n",
      "To be able to connect to a MySQL database, SQLAlchemy requires a supported\n",
      "MySQL client package such as pymysql to be installed. This package can be added to\n",
      "the docker.txt requirements file.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 17e to check out this version of the applica‐\n",
      "tion.\n",
      "The change made to the requirements/docker.txt file requires the container image to\n",
      "be rebuilt:\n",
      "$ docker build -t flasky:latest .\n",
      "264 \n",
      "| \n",
      "Chapter 17: Deployment\n",
      "\n",
      "If you are still running the previous application container, stop it and remove it with\n",
      "docker rm -f. Then start a new container with the updated application:\n",
      "$ docker run -d -p 8000:5000 --link mysql:dbserver \\\n",
      "  -e DATABASE_URL=mysql+pymysql://flasky:<database-password>@dbserver/flasky \\\n",
      "  -e MAIL_USERNAME=<your-gmail-username> -e MAIL_PASSWORD=<your-gmail-password> \\\n",
      "  flasky:latest\n",
      "There are two additions to the docker run command shown here. The --link option\n",
      "configures a connection between the new container and another existing one. The\n",
      "argument to --link consists of two names separated by a colon: the source container\n",
      "name or ID, and an alias for that container in the container being created. In this\n",
      "example the source container is mysql, the database container started earlier. This\n",
      "container is going to be accessible in the new Flasky container with the dbserver\n",
      "hostname.\n",
      "To complete the configuration, a DATABASE_URL environment variable is added, with a\n",
      "connection URL that points to the flasky database in the mysql container. The\n",
      "dbserver alias is used as the database host, as Docker makes sure that this name\n",
      "resolves to the IP address of the linked container. The value of the MYSQL_PASSWORD\n",
      "environment variable set in the mysql container must be included in the connection\n",
      "URL for this container as well. The value of DATABASE_URL overrides the default\n",
      "SQLite database, so with this simple change the container will be configured to con‐\n",
      "nect to the MySQL database.\n",
      "The Docker Hub repository is a gold mine of very useful applica‐\n",
      "tions and services that are packaged and ready to use in a Docker\n",
      "environment, either standalone or as base images for your own\n",
      "containers. You will find that all sorts of projects (including data‐\n",
      "bases, web servers, load balancers, programming languages, operat‐\n",
      "ing systems, and more) offer official images.\n",
      "Container Orchestration with Docker Compose\n",
      "Containerized applications are usually composed of several running containers. You\n",
      "have seen in the previous section that the main application and the database server\n",
      "run in independent containers. As the application grows in complexity, it will invaria‐\n",
      "bly need more containers. Some applications are going to require additional services,\n",
      "such as message queues or caches. Other applications may take advantage of a micro‐\n",
      "services architecture and have a distributed structure with several smaller sub-\n",
      "applications, each running in its own container. Applications that have to handle high\n",
      "loads or need to be fault-tolerant will want to scale out by running several instances\n",
      "behind a load balancer.\n",
      "Docker Containers \n",
      "| \n",
      "265\n",
      "\n",
      "As the number of containers that are part of the application increases, the task of\n",
      "managing and coordinating all these containers is going to become much harder if\n",
      "Docker alone is used. Container orchestration frameworks built on top of Docker help\n",
      "with this task.\n",
      "The Compose toolset provides basic orchestration functionality, included with the\n",
      "Docker installation. With Compose, the containers that are part of an application are\n",
      "described in a configuration file, typically named docker-compose.yml. The docker-\n",
      "compose command can then start all the containers associated with the application\n",
      "using a single command.\n",
      "Example 17-12 shows a docker-compose.yml file that represents the containerized Fla‐\n",
      "sky along with its MySQL service.\n",
      "Example 17-12. docker-compose.yml: compose configuration\n",
      "version: '3'\n",
      "services:\n",
      "  flasky:\n",
      "    build: .\n",
      "    ports:\n",
      "      - \"8000:5000\"\n",
      "    env_file: .env\n",
      "    links:\n",
      "      - mysql:dbserver\n",
      "    restart: always\n",
      "  mysql:\n",
      "    image: \"mysql/mysql-server:5.7\"\n",
      "    env_file: .env-mysql\n",
      "    restart: always\n",
      "This file is written in YAML, which is a clean and simple format that can represent\n",
      "hierarchical structures that are composed of key-value maps and lists. The version\n",
      "key specifies which version of Compose is used, and the services key defines the\n",
      "containers of the application as its children. In the case of Flasky, these are two serv‐\n",
      "ices named flasky and mysql.\n",
      "For services such as flasky, which are built as part of the application, the subkeys\n",
      "specify the arguments that are given to the docker build and docker run com‐\n",
      "mands. The build key specifies the build directory, where the Dockerfile is located.\n",
      "The ports key specifies the network port mappings. The env_file key is a conve‐\n",
      "nient way to define several environment variables that the container needs. The links\n",
      "key establishes a link to the MySQL container, by exposing it with the hostname\n",
      "dbserver. The restart key set to always provides a simple way for Docker to auto‐\n",
      "matically restart the container if it exits unexpectedly. The .env file for this deploy‐\n",
      "ment should have the following variables in it:\n",
      "266 \n",
      "| \n",
      "Chapter 17: Deployment\n",
      "\n",
      "FLASK_APP=flasky.py\n",
      "FLASK_CONFIG=docker\n",
      "SECRET_KEY=3128b4588e7f4305b5501025c13ceca5\n",
      "MAIL_USERNAME=<your-gmail-username>\n",
      "MAIL_PASSWORD=<your-gmail-password>\n",
      "DATABASE_URL=mysql+pymysql://flasky:<database-password>@dbserver/flasky\n",
      "The mysql service has a simpler structure, because this is a service that is started from\n",
      "a stock image that does not require a build step. The image key specifies the name\n",
      "and tag of the container image to use for this service. As with the docker run com‐\n",
      "mand, Docker will download this image from the container image registry. The\n",
      "env_file and restart keys are similar to those used in the flasky container. Note\n",
      "how the environment variables for the MySQL container are stored in a separate file\n",
      "named .env-mysql. While it would be easier to add the environment variables needed\n",
      "by all containers to the .env file, it is a good practice to prevent one container from\n",
      "having access to the secrets of another. The .env-mysql file needs the following envi‐\n",
      "ronment variables defined:\n",
      "MYSQL_RANDOM_ROOT_PASSWORD=yes\n",
      "MYSQL_DATABASE=flasky\n",
      "MYSQL_USER=flasky\n",
      "MYSQL_PASSWORD=<database-password>\n",
      "The .env and .env-mysql files contain passwords and other sensitive\n",
      "information, so they should never be added to source control.\n",
      "A complete reference for the docker-compose.yml file is found at\n",
      "“the Docker website”.\n",
      "A typical problem with orchestrated systems is that containers are started in the\n",
      "wrong order—or in the correct order, but without giving the containers for base serv‐\n",
      "ices enough time to start and initialize before starting higher-level containers that\n",
      "depend on them. In the case of Flasky, the mysql container needs to start first, so that\n",
      "the database is up and running when the flasky container starts. Then it can connect\n",
      "to the database, apply the database migrations, and finally start the web server.\n",
      "Compose will start the mysql and flasky containers in the right order, because it will\n",
      "detect the dependency between them from the links key in the flasky container. But\n",
      "Compose is not going to wait for MySQL to start, which might take a few seconds.\n",
      "When designing distributed systems, it is a good practice to implement retries in all\n",
      "connections to external services. Example 17-13 shows how the boot.sh script that\n",
      "Docker Containers \n",
      "| \n",
      "267\n",
      "\n",
      "starts the flasky container can be made more robust by retrying the flask deploy\n",
      "command, which retries the database upgrade until it succeeds.\n",
      "Example 17-13. boot.sh: waiting for the database to be up\n",
      "#!/bin/sh\n",
      "source venv/bin/activate\n",
      "while true; do\n",
      "    flask deploy\n",
      "    if [[ \"$?\" == \"0\" ]]; then\n",
      "        break\n",
      "    fi\n",
      "    echo Deploy command failed, retrying in 5 secs...\n",
      "    sleep 5\n",
      "done\n",
      "exec gunicorn -b :5000 --access-logfile - --error-logfile - flasky:app\n",
      "By running flask deploy inside a retry loop, the container will be able to tolerate\n",
      "failures due to the database service not being immediately ready to accept requests.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 17f to check out this version of the applica‐\n",
      "tion. Also make sure that the .env and .env-mysql environment files\n",
      "are created and populated with values correct for your environ‐\n",
      "ment.\n",
      "Now that the Compose configuration is complete, the application can be started with\n",
      "the docker-compose up command:\n",
      "$ docker-compose up -d --build\n",
      "The --build option to docker-compose up indicates that a build step should run\n",
      "before launching the application. This will cause the flasky container image to be\n",
      "built. After the image is created, the mysql and flasky containers will be started in\n",
      "that order. The -d option starts the containers in detached mode, as with the single\n",
      "container. After a few seconds, the application should be up and running in the back‐\n",
      "ground, and you should be able to connect to it at http://localhost:8000.\n",
      "Compose consolidates the logging from all the containers into a single stream, which\n",
      "you can see with the docker-compose logs command:\n",
      "$ docker-compose logs\n",
      "Or, if you want to constantly monitor the log stream:\n",
      "$ docker-compose logs -f\n",
      "268 \n",
      "| \n",
      "Chapter 17: Deployment\n",
      "\n",
      "The docker-compose ps command shows a summary of all the application contain‐\n",
      "ers that are running and their state:\n",
      "$ docker-compose ps\n",
      "     Name                Command          State           Ports\n",
      "------------------------------------------------------------------------\n",
      "flasky_flasky_1   ./boot.sh               Up      0.0.0.0:8000->5000/tcp\n",
      "flasky_mysql_1    /entrypoint.sh mysqld   Up      3306/tcp, 33060/tcp\n",
      "To upgrade an application to a new version, simply make the necessary changes to it\n",
      "and repeat the docker-compose up command used previously to start it. Compose\n",
      "will rebuild the application container if anything changed, and then replace the older\n",
      "container with a fresh one.\n",
      "To stop the application, use the docker-compose down command, or docker-compose\n",
      "rm --stop --force if you also want to remove the stopped containers.\n",
      "Cleaning Up Old Containers and Images\n",
      "As you work with containers, your system will invariably accumulate old containers\n",
      "or images that are not needed anymore. It is a good idea to routinely review and clean\n",
      "those up, so that they don’t take up space on the system.\n",
      "To see the list of containers in the system, use the following command:\n",
      "$ docker ps -a\n",
      "This will show containers that are running, and containers that were stopped but are\n",
      "still in the system. To delete any containers from this list, use the docker rm -f com‐\n",
      "mand and provide the names or IDs to remove:\n",
      "$ docker rm -f <name-or-id> <name-or-id> ...\n",
      "To see the list of container images stored in your system, use the docker images\n",
      "command. If there are any images that you want to remove, you can do so with the\n",
      "docker rmi command.\n",
      "Some containers create virtual volumes on the host computer that are used for storage\n",
      "outside of the container filesystem. The MySQL container image, for example, puts all\n",
      "the database files in a volume. You can view a list of all the allocated volumes in your\n",
      "system with docker volume ls. To remove a volume that is unused, use docker\n",
      "volume rm.\n",
      "If you prefer a more automatic cleanup, the docker system prune --volumes com‐\n",
      "mand will remove any unused images or volumes, and any stopped containers that\n",
      "are still in the system.\n",
      "Docker Containers \n",
      "| \n",
      "269\n",
      "\n",
      "Using Docker in Production\n",
      "Many people consider Docker a development and testing platform only. While the\n",
      "techniques presented in the previous sections can be used to deploy applications on\n",
      "production servers running Docker, there are some limitations and security concerns\n",
      "that need to be considered:\n",
      "Monitoring and alerting\n",
      "What happens if a containerized application crashes? Docker can restart a con‐\n",
      "tainer that exits unexpectedly, but it will not monitor your containers, nor will it\n",
      "send alerts when they behave erratically.\n",
      "Logging\n",
      "Docker maintains a separate log stream for each container. Compose improves\n",
      "this by offering a consolidated stream, but without long-term storage or search‐\n",
      "ing and filtering capabilities.\n",
      "Management of secrets\n",
      "Configuring passwords and other credentials through environment variables is\n",
      "insecure, since Docker exposes pre-configured environment variables via the\n",
      "docker inspect command or through its API.\n",
      "Reliability and scaling\n",
      "To help with fault tolerance, or to accommodate increasing load demands, it is\n",
      "necessary to run several instances of the application on several hosts and behind\n",
      "one or more load balancers.\n",
      "These limitations are generally addressed by more elaborated orchestration frame‐\n",
      "works built on top of Docker or other container runtimes. Frameworks such as\n",
      "Docker Swarm (now incorporated into Docker), Apache Mesos, and Kubernetes are\n",
      "good choices for building robust container deployments.\n",
      "Traditional Deployments\n",
      "So far you have seen how Heroku and Docker manage deployments. To complete this\n",
      "review of deployment strategies, this section will describe a traditional hosting\n",
      "option, which involves buying or renting a server, either physical or virtual, and man‐\n",
      "ually setting up all the required components on it. This is obviously the most labori‐\n",
      "ous option of all, but it can be a convenient option when you have terminal access to\n",
      "production server hardware. The following sections will give you an idea of the work\n",
      "involved.\n",
      "270 \n",
      "| \n",
      "Chapter 17: Deployment\n",
      "\n",
      "Server Setup\n",
      "There are several administration tasks that must be performed on the server before it\n",
      "can host applications:\n",
      "• Install a database server such as MySQL or Postgres. Using an SQLite database is\n",
      "also possible but is not recommended for a production server due to its many\n",
      "limitations with regard to modification of existing database schemas.\n",
      "• Install a Mail Transport Agent (MTA) such as Sendmail or Postfix to send email\n",
      "out to users. Using Gmail in a production application is not possible, as this ser‐\n",
      "vice has very restrictive quotas and specifically prohibits commercial use in its\n",
      "terms of service.\n",
      "• Install a production-ready web server such as Gunicorn or uWSGI.\n",
      "• Install a process-monitoring utility such as Supervisor, that immediately restarts\n",
      "the web server if it crashes or after the host is power-cycled.\n",
      "• Install and configure an SSL certificate to enable secure HTTP.\n",
      "• (Optional but highly recommended) Install a front-end reverse proxy web server\n",
      "such as nginx or Apache. This server is configured to serve static files directly and\n",
      "forward application requests into the application’s web server, which is listening\n",
      "on a private port on localhost.\n",
      "• Secure the server. This includes several tasks that have the goal of reducing vul‐\n",
      "nerabilities in the server such as installing firewalls, removing unused software\n",
      "and services, and so on.\n",
      "Instead of manually performing these tasks, create a scripted\n",
      "deployment using an automation framework such as Ansible, Chef,\n",
      "or Puppet.\n",
      "Importing Environment Variables\n",
      "Similarly to Heroku and Docker, an application running on a standalone server relies\n",
      "on certain settings such as the database connection URL, email server credentials, etc.\n",
      "being provided in environment variables.\n",
      "Because there is no Heroku or Docker to configure these variables before the applica‐\n",
      "tion starts, the procedure to set the variables is dependent on the platform and tools\n",
      "used. To make the configuration of environment variables easier and uniform across\n",
      "deployment platforms, the short code block in Example 17-14 imports into the envi‐\n",
      "ronment a .env file similar to the one used with the heroku local and docker-\n",
      "Traditional Deployments \n",
      "| \n",
      "271\n",
      "\n",
      "compose commands, using a Python package called python-dotenv that needs to be\n",
      "installed with pip. This is done in flasky.py before the application instance is created,\n",
      "so that by the time the configuration is imported these variables are accessible in the\n",
      "environment.\n",
      "Example 17-14. flasky.py: importing the environment from the .env file\n",
      "import os\n",
      "from dotenv import load_dotenv\n",
      "dotenv_path = os.path.join(os.path.dirname(__file__), '.env')\n",
      "if os.path.exists(dotenv_path):\n",
      "    load_dotenv(dotenv_path)\n",
      "The .env file can define the FLASK_CONFIG variable that selects the configuration to\n",
      "use, the DATABASE_URL connection, the email server credentials, etc. As explained\n",
      "before, a .env file should not be added to source control due to the sensitive nature of\n",
      "some of the items in it.\n",
      "If you created a .env file for use with Heroku or Docker, review it\n",
      "and adjust it appropriately, because with the changes just made, the\n",
      "application will import the variables defined in this file for all con‐\n",
      "figurations.\n",
      "Setting Up Logging\n",
      "For Unix-based servers, logging can be sent to the syslog daemon. A new configura‐\n",
      "tion specifically for Unix can be created as a subclass of ProductionConfig, as shown\n",
      "in Example 17-15.\n",
      "Example 17-15. config.py: Unix example configuration\n",
      "class UnixConfig(ProductionConfig):\n",
      "    @classmethod\n",
      "    def init_app(cls, app):\n",
      "        ProductionConfig.init_app(app)\n",
      "        # log to syslog\n",
      "        import logging\n",
      "        from logging.handlers import SysLogHandler\n",
      "        syslog_handler = SysLogHandler()\n",
      "        syslog_handler.setLevel(logging.WARNING)\n",
      "        app.logger.addHandler(syslog_handler)\n",
      "With this configuration, application logs will be written to the configured syslog mes‐\n",
      "sages file, typically /var/log/messages or /var/log/syslog depending on the Linux distri‐\n",
      "272 \n",
      "| \n",
      "Chapter 17: Deployment\n",
      "\n",
      "bution. The syslog service can be configured to write a separate log file for application\n",
      "logs, or to send the logs to a different machine if desired.\n",
      "If you have cloned the application’s Git repository on GitHub, you\n",
      "can run git checkout 17g to check out this version of the applica‐\n",
      "tion.\n",
      "Traditional Deployments \n",
      "| \n",
      "273\n",
      "\n",
      "\n",
      "CHAPTER 18\n",
      "Additional Resources\n",
      "You are pretty much done with this book. Congratulations! I hope the topics that I\n",
      "have covered have given you a solid base to begin building your own applications\n",
      "with Flask. The code examples are open source and have a permissive license, so you\n",
      "are welcome to use as much of my code as you want to seed your projects, even if\n",
      "they are of a commercial nature. In this short final chapter, I want to give you a list of\n",
      "additional tips and resources that might be useful as you continue working with\n",
      "Flask.\n",
      "Using an Integrated Development Environment (IDE)\n",
      "Developing Flask applications in an integrated development environment (IDE) can\n",
      "be very convenient, since features such as code completion and an interactive debug‐\n",
      "ger can speed up the coding process considerably. Some of the IDEs that work well\n",
      "with Flask are listed here:\n",
      "PyCharm\n",
      "IDE from JetBrains with Community (free) and Professional (paid) editions,\n",
      "both compatible with Flask applications. Available on Linux, macOS, and Win‐\n",
      "dows.\n",
      "Visual Studio Code\n",
      "Open source IDE from Microsoft. A third-party Python plug-in must be installed\n",
      "to have access to code completion and debugging features with Flask applica‐\n",
      "tions. Available on Linux, macOS, and Windows.\n",
      "PyDev\n",
      "Open source IDE based on Eclipse. Available on Linux, macOS, and Windows.\n",
      "275\n",
      "\n",
      "Finding Flask Extensions\n",
      "The examples in this book rely on several extensions and packages, but there are\n",
      "many more that are also useful and were not discussed. The following is a short list of\n",
      "some additional packages that are worth exploring:\n",
      "• Flask-Babel: Internationalization and localization support\n",
      "• Marshmallow: Serialization and deserialization of Python objects, useful for API\n",
      "resource representations\n",
      "• Celery: Task queue for processing background jobs\n",
      "• Frozen-Flask: Conversion of a Flask application to a static website\n",
      "• Flask-DebugToolbar: In-browser debugging tools\n",
      "• Flask-Assets: Merging, minifying, and compiling of CSS and JavaScript assets\n",
      "• Flask-Session: Alternative implementation of user sessions that use server-side\n",
      "storage\n",
      "• Flask-SocketIO: Socket.IO server implementation with support for WebSocket\n",
      "and long-polling\n",
      "If the functionality that you need for your project is not covered by any of the exten‐\n",
      "sions and packages mentioned in this book, then your first destination to look for\n",
      "additional extensions should be the official Flask Extension Registry. Other good\n",
      "places to search are the Python Package Index, GitHub, and Bitbucket.\n",
      "Getting Help\n",
      "If you reach a point where you are blocked by an issue that you cannot resolve on\n",
      "your own, you should keep in mind that there is a community of Flask developers\n",
      "like you that will be happy to help you.\n",
      "A great place to ask questions about Flask or any related extensions is Stack Overflow.\n",
      "Other developers that see your question and know how to answer will post their\n",
      "answers, which are voted up or down according to their quality. You, as the owner of\n",
      "the question, can then select the best answer. All questions and their answers stay on\n",
      "the site and appear in search results. So, by asking your question on this platform,\n",
      "you help grow the collection of information about Flask.\n",
      "Reddit also has a friendly Flask-dedicated subreddit where you can post questions.\n",
      "Finally, if you use IRC, the #pocoo channel on Freenode is frequented by Flask devel‐\n",
      "opers of all levels who can help you one-on-one with your problem.\n",
      "276 \n",
      "| \n",
      "Chapter 18: Additional Resources\n",
      "\n",
      "Getting Involved with Flask\n",
      "Flask would not be as awesome as it is without the work done by its community of\n",
      "developers. As you are now becoming part of this community and benefiting from\n",
      "the work of so many volunteers, you should consider finding a way to give something\n",
      "back. Here are some ideas to help you get started:\n",
      "• Review the documentation for Flask or your favorite related project and submit\n",
      "corrections or improvements.\n",
      "• Translate the documentation into a new language.\n",
      "• Answer questions on Q&A sites such as Stack Overflow.\n",
      "• Talk about your work with your peers at user group meetings or conferences.\n",
      "• Contribute bug fixes or improvements to packages that you use.\n",
      "• Write new Flask extensions and release them as open source.\n",
      "• Release your applications as open source.\n",
      "I hope you decide to volunteer in one of these ways, or any others that are meaningful\n",
      "to you. If you do, thank you!\n",
      "Getting Involved with Flask \n",
      "| \n",
      "277\n",
      "\n",
      "\n",
      "Index\n",
      "A\n",
      "abort function, 22, 231\n",
      "absolute URLs (in links), 37\n",
      "account confirmation, 118-125\n",
      "generating confirmation tokens with its‐\n",
      "dangerous, 118\n",
      "sending confirmation emails, 120-124\n",
      "account management, 125\n",
      "role assignment to users, 135\n",
      "activation, virtual environments, 4\n",
      "administrator roles, 127\n",
      "assignment of, 131\n",
      "administrators, user profile editor for, 143-145\n",
      "admin_required decorator, 145\n",
      "after_app_request hook, 238\n",
      "after_request hook, 20\n",
      "Alembic database migrations, 74\n",
      "Alpine Linux, 258\n",
      "AnonymousUser custom class, 132\n",
      "Apache Mesos, 270\n",
      "APIs (application programming interfaces),\n",
      "199-218\n",
      "introduction to REST, 199-203\n",
      "resources, 200\n",
      "versioning web services, 202\n",
      "RESTful web services with Flask, 203-218\n",
      "app.add_url_rule method, 8, 19\n",
      "app.cli.command decorator, 96\n",
      "app.config object, 44\n",
      "app.run method, 11\n",
      "app.shell_context_processor decorator, 72\n",
      "application context, 18\n",
      "and email on background thread, 83\n",
      "application directory, creating, 2\n",
      "application instance, 7\n",
      "application structure, basic, 7-23\n",
      "command-line options, 15\n",
      "complete application example, 9\n",
      "debug mode, 13\n",
      "development server, 10\n",
      "dynamic routes, 12\n",
      "Flask extensions, 23\n",
      "initializing applications, 7\n",
      "request-response cycle, 17-22\n",
      "routes and view functions, 8\n",
      "applications, large, structure of, 85-97\n",
      "application package, 88-92\n",
      "implementing application functionality\n",
      "in a blueprint, 90-92\n",
      "using an application factory, 88-89\n",
      "application script, 93\n",
      "configuration options, 86-88\n",
      "database setup, 96\n",
      "project structure, 85\n",
      "requirements file, 93\n",
      "running the application, 97\n",
      "unit tests, 94\n",
      "app_errorhandler decorator, 91, 205\n",
      "association tables (database), 65, 172\n",
      "followers table as a model, 174\n",
      "auth.login_required decorator, 208\n",
      "authentication, 101-125\n",
      "account confirmation, 118-125\n",
      "account management, 125\n",
      "creating blueprint for, 105\n",
      "Flask extensions for, 101\n",
      "Flask-Mail Gmail account, 80\n",
      "in API blueprint, 204\n",
      "279\n",
      "\n",
      "new user registration, 115-118\n",
      "password security, 102-105\n",
      "token-based, 208-210\n",
      "user authentication with Flask-Login,\n",
      "107-115\n",
      "with Flask-HTTPAuth, 206\n",
      "with Heroku account, 245\n",
      "automation frameworks, 271\n",
      "avatars, 146-149\n",
      "avatar in profile page, 147\n",
      "blog post author, 153\n",
      "Gravatar query string arguments, 146\n",
      "Gravatar URL generation, 146\n",
      "AWS EC2, 243\n",
      "B\n",
      "background jobs, for sending email, 84\n",
      "background thread, email on, 83\n",
      "bash, 2\n",
      "before_app_request decorator, 122, 138\n",
      "before_first_request hook, 20\n",
      "before_request hook, 20, 122\n",
      "before_request handler with authentication,\n",
      "208\n",
      "Bleach package, 164\n",
      "blocks (in base templates), 29\n",
      "available blocks in Flash-Bootstrap, 32\n",
      "blog posts, 151-169\n",
      "editor for, 167-169\n",
      "on profile pages, 154\n",
      "paginating long lists of, 155-161\n",
      "permanent links to, 165-167\n",
      "querying followed posts using database join,\n",
      "181-183\n",
      "rich text posts with Markdown and Flask-\n",
      "PageDown, 161-165\n",
      "showing followed posts on home page,\n",
      "183-187\n",
      "submission and display of, 151-154\n",
      "blueprints, 90-92\n",
      "creating, 90\n",
      "creating authentication blueprint, 105\n",
      "creating for RESTful API, 203\n",
      "error handlers in, 91\n",
      "registration with app in factory function, 91\n",
      "RESTful API blueprint registration, 204\n",
      "routes in, 91\n",
      "BooleanField class, 109\n",
      "boot.sh (container startup script), 259\n",
      "Bootstrap, integration using Flash-Bootstrap,\n",
      "30-33\n",
      "business logic, 25\n",
      "C\n",
      "cardinality, 58\n",
      "Click package, 1\n",
      "cloud, deployment to, 243\n",
      "code coverage reports, 221-224\n",
      "code examples from this book, xiii\n",
      "collections (NoSQL databases, 58\n",
      "columns (database), 57\n",
      "db.Column class, 62\n",
      "SQLAlchemy column options, 63\n",
      "SQLAlchemy column types, 63\n",
      "command-line interface (CLI), installing for\n",
      "Heroku, 245\n",
      "command-line options, flask command, 15\n",
      "comments (user) (see user comments)\n",
      "committing database sessions, 67\n",
      "conditional statements (in templates), 28, 48\n",
      "Config class, 87\n",
      "configuration\n",
      "applications created by application script, 93\n",
      "applications created by factory function, 89\n",
      "applications deployed on Heroku platform,\n",
      "247\n",
      "configuring email for applications on Her‐\n",
      "oku, 248\n",
      "Docker deployment, 258\n",
      "for slow query reporting, 238\n",
      "large applications, options for, 86-88\n",
      "sending email for application errors, 242\n",
      "Unix-based servers, logging, 272\n",
      "containers, 244, 256\n",
      "(see also Docker)\n",
      "content negotiation, 205\n",
      "context processors, 134\n",
      "contexts, 17\n",
      "for email on background thread, 83\n",
      "shell context, adding, 72\n",
      "control structures (Jinja2), 28\n",
      "cookies\n",
      "client-side, user sessions in, 52\n",
      "setting in response object, 21\n",
      "showing followed posts, 184\n",
      "Coordinated Universal Time (UTC), 38\n",
      "coverage tool, 222\n",
      "(see also code coverage reports)\n",
      "280 \n",
      "| \n",
      "Index\n",
      "\n",
      "create_app factory function\n",
      "attaching authentication blueprint to app,\n",
      "106\n",
      "cross-site request forgery (CSRF) attacks, 44\n",
      "CSS\n",
      "Bootstrap CSS files, 30, 33\n",
      "Bootstrap pagination classes, 159\n",
      "classes for avatar in profile page, 147\n",
      "styles for blog posts, 153\n",
      "cURL, 217\n",
      "current_time variable, 39\n",
      "current_user context variable, 113, 142\n",
      "current_user.can function, 132\n",
      "current_user.is_administrator function, 132\n",
      "current_user.is_authenticated property, 110,\n",
      "114\n",
      "current_user._get_current_object method, 152\n",
      "Cygwin, 2\n",
      "D\n",
      "database (in database URLs), 61\n",
      "databases, 57-77\n",
      "creating tables, 66\n",
      "deleting rows, 68\n",
      "Flask support for, xi\n",
      "inserting rows, 66\n",
      "integration with Python shell, 72\n",
      "large applications using different databases,\n",
      "87\n",
      "logging slow performance, 237-239\n",
      "making users their own followers in the\n",
      "database, 186\n",
      "management with Flask-SQLAlchemy, 61\n",
      "migrations with Flask-Migrate, 73-77\n",
      "adding more migrations, 76\n",
      "upgrading the database, 75\n",
      "model definition, 62-64\n",
      "modifying rows, 68\n",
      "NoSQL, 58\n",
      "of user roles, 127-131\n",
      "adding new roles in shell session, 134\n",
      "Post model for blog posts, 151\n",
      "Markdown text handling, 164\n",
      "provisioning a database on Heroku, 246\n",
      "Python database frameworks, 59-60\n",
      "querying followed posts using database join,\n",
      "181-183\n",
      "filter_by query filter, 182\n",
      "join query filter, 182\n",
      "querying rows, 68\n",
      "relational model, 57\n",
      "relationships in, 64-65, 171-178\n",
      "association table, 172\n",
      "representation of blog post comments,\n",
      "189-191\n",
      "setup in a large application, 96\n",
      "SQL, 57\n",
      "SQL vs. NoSQL, 59\n",
      "storing MD5 hashes for user avatars, 148\n",
      "use in view functions, 71-72\n",
      "user information in, 137\n",
      "user, password hashes stored in, 102\n",
      "using an external database with Docker con‐\n",
      "tainers, 264\n",
      "DATABASE_URL environment variable, 246,\n",
      "265\n",
      "DataRequired form field validator, 45, 49, 109\n",
      "dates and time\n",
      "last visit date for users, 138\n",
      "localization with Flask-Moment, 38\n",
      "timestamps in user information database,\n",
      "137\n",
      "db object, 62\n",
      "db.Column class, 62\n",
      "db.create_all function, 66, 75\n",
      "db.ForeignKey function, 64\n",
      "db.relationship function, 64\n",
      "db.session object, 67\n",
      "db.session.add method, 68\n",
      "db.session.commit method, 67, 121\n",
      "db.session.delete method, 68\n",
      "db.session.rollback method, 68\n",
      "debugging\n",
      "debug mode, 13, 93\n",
      "--debugger and --no-debugger command-\n",
      "line options, 16\n",
      "of errors during production, 242\n",
      "debugging subsystem, 1\n",
      "decorators, 8\n",
      "custom, checking user permissions, 132\n",
      "order of, in view functions using multiple\n",
      "decorators, 133\n",
      "request hooks implemented as, 20\n",
      "DELETE request method (HTTP), 201\n",
      "denormalization (NoSQL databases), 59\n",
      "dependencies, 1\n",
      "for applications deployed on Heroku, 248\n",
      "for development vs. production, 156\n",
      "Index \n",
      "| \n",
      "281\n",
      "\n",
      "installing into virtual environment with pip,\n",
      "5\n",
      "requirements file for large applications, 93\n",
      "deploy command, 241\n",
      "deployment, 241-273\n",
      "Docker containers, 256-270\n",
      "in the cloud, 243\n",
      "logging errors during production, 242\n",
      "on Heroku platform, 244-255\n",
      "preparing the application, 244-253\n",
      "traditional, 270-273\n",
      "importing environment variables, 271\n",
      "server setup, 271\n",
      "setting up logging, 272\n",
      "workflow, 241\n",
      "deserialization (APIs), 212\n",
      "development web server, 10\n",
      "DEV_DATABASE_URL environment variable,\n",
      "96\n",
      "Docker, 244, 256-270\n",
      "building a Docker image, 257\n",
      "container orchestration with Compose,\n",
      "265-269\n",
      "Dockerfile, 257\n",
      "images command, 260, 269\n",
      "installing, 256\n",
      "login command, 263\n",
      "logs command, 262, 264\n",
      "ps command, 269\n",
      "push command, 263\n",
      "rm command, 262, 269\n",
      "rmi command, 269\n",
      "run command, 261, 264\n",
      "stop command, 262\n",
      "Swarm, 270\n",
      "system command, 269\n",
      "tag command, 263\n",
      "using an external database, 264\n",
      "using in production, 270\n",
      "version command, 256\n",
      "volume command, 269\n",
      "Docker Compose, 265\n",
      "docker-compose.yml file, 266\n",
      "logs command, 268\n",
      "ps command, 269\n",
      "up command, 268\n",
      "Docker Hub, 263\n",
      "applications and services on, 265\n",
      "document-oriented databases, 57\n",
      "dynamic routes, 9\n",
      "dynamic URLs, generating with url_for func‐\n",
      "tion, 37\n",
      "dynos (Heroku), 244\n",
      "E\n",
      "EC2 service (AWS), 243\n",
      "email, 79\n",
      "(see also Flask-Mail)\n",
      "configuring for applications on Heroku, 248\n",
      "configuring for Docker container, 261\n",
      "confirming user accounts, 120-124\n",
      "handling address changes for user accounts,\n",
      "125\n",
      "sending for application errors, 242\n",
      "entity-relationship diagrams, 58\n",
      ".env file, 253, 266-267, 271\n",
      "environment variables\n",
      "defined in Dockerfile, 258\n",
      "importing in traditional deployment, 271\n",
      "in large application configuration, 87\n",
      "EqualTo form field validator, 116\n",
      "error handling\n",
      "API error handler for ValidationError, 212\n",
      "error handlers in app blueprints, 91\n",
      "Flask-HTTPAuth error handler, 208\n",
      "in RESTful web services, 204\n",
      "with content negotiation, 205\n",
      "error pages, custom, 33-36\n",
      "errors, logging during production, 242\n",
      "extends directive, 30, 32\n",
      "extensibility of Flask, xi\n",
      "extensions, 1, 23, 30\n",
      "additional Flask extensions and packages,\n",
      "276\n",
      "initialization, 31\n",
      "F\n",
      "factory function, creating applications with, 88\n",
      "fake blog post data, creating, 155-157\n",
      "Faker package, 155\n",
      "filters (database), 27\n",
      "offset() query filter, 157\n",
      "query of followed posts using database join,\n",
      "183\n",
      "using with database queries, 68\n",
      "filter_by method, 69\n",
      "SQLAlchemy filters for, 69\n",
      "flash function, 53\n",
      "282 \n",
      "| \n",
      "Index\n",
      "\n",
      "Flask\n",
      "application factory function, 88\n",
      "app_errorhandler decorator, 91, 205\n",
      "basic multiple-file application structure, 85\n",
      "custom commands, 96\n",
      "dynamic routes, 9\n",
      "extensions, 30\n",
      "Flask class, 7\n",
      "flask command options, 15\n",
      "installing into virtual environment with pip,\n",
      "5\n",
      "server shutdown, 230\n",
      "test client, 224-229\n",
      "working with, additional resources for,\n",
      "275-277\n",
      "flask db downgrade command, 76\n",
      "flask db migrate command, 75-76\n",
      "flask db stamp command, 76\n",
      "flask db upgrade command, 75-76\n",
      "flask deploy command, 268\n",
      "Flask Extension Registry, 276\n",
      "flask run command, 10\n",
      "--host argument, 16\n",
      "options, 16\n",
      "flask shell command, 15, 66, 72\n",
      "flask test command, --coverage option, 222\n",
      "Flask-Bootstrap, 30-33\n",
      "blocks, 29\n",
      "initialization, 30\n",
      "installing, 30\n",
      "quick_form macro, 47\n",
      "wtf.quick_form Jinja2 macro, 110\n",
      "wtf.quick_form method, 47\n",
      "Flask-HTTPAuth, 206\n",
      "before_request handler with authentication,\n",
      "208\n",
      "error handler, 208\n",
      "initialization, 207\n",
      "token-based authentication support, 209\n",
      "Flask-Login, 107-115\n",
      "adding a login form, 109\n",
      "AnonymousUserMixin class, 132\n",
      "current_user context variable, 113, 152\n",
      "how it works, 113\n",
      "LoginManager class, 108\n",
      "login_manager.anonymous_user attribute,\n",
      "132\n",
      "login_required decorator, 108, 114\n",
      "login_user function, 111\n",
      "logout_user function, 113\n",
      "preparing User model for logins, 107\n",
      "signing users in, 111\n",
      "signing users out, 112\n",
      "testing logins, 114\n",
      "UserMixin class, 107\n",
      "user_loader decorator, 108\n",
      "Flask-Mail, 79-84\n",
      "initialization, 80\n",
      "integrating emails with the application, 81\n",
      "sending asynchronous email, 83\n",
      "sending email from Python shell, 81\n",
      "sending email through Gmail, 80\n",
      "SMTP server configuration keys, 79\n",
      "Flask-Migrate, 73-77\n",
      "adding more migrations, 76\n",
      "considerations in changing database sche‐\n",
      "mas, 74\n",
      "creating or upgrading database tables in\n",
      "large application, 96\n",
      "initialization, 73\n",
      "upgrading the database, 75\n",
      "Flask-Moment, 38\n",
      "format function, 40\n",
      "fromNow function, 40\n",
      "locale function, 41\n",
      "Flask-PageDown, 162-164\n",
      "initialization, 162\n",
      "Markdown-enabled post form, 162\n",
      "Flask-SQLAlchemy, 60\n",
      "add session method, 67-68\n",
      "column options, 63\n",
      "create_all method, 66, 95\n",
      "database management with, 61\n",
      "delete session method, 68\n",
      "drop_all method, 66\n",
      "enabling recording of query statistics, 239\n",
      "filter_by query filter, 71\n",
      "first_or_404 query method, 139\n",
      "get_debug_queries function, 237\n",
      "get_or_404 convenience function, 145\n",
      "models, 62\n",
      "MySQL configuration, 61\n",
      "paginate method, 158\n",
      "pagination object attributes, 158\n",
      "pagination object methods, 159\n",
      "Postgres configuration, 61\n",
      "query executors, 69\n",
      "query filters, 69\n",
      "Index \n",
      "| \n",
      "283\n",
      "\n",
      "query object (database), 68\n",
      "query statistics recorded by, 238\n",
      "querying followed posts using databse join,\n",
      "182\n",
      "SQLALCHEMY_DATABASE_URI configu‐\n",
      "ration, 61\n",
      "SQLALCHEMY_TRACK_MODIFICA‐\n",
      "TIONS configuration, 61\n",
      "SQLite configuration, 61\n",
      "Flask-SSLify, 249\n",
      "Flask-WTF, 43\n",
      "BooleanField class, 109\n",
      "configuration, 44\n",
      "cross-site request forgery (CSRF), 44\n",
      "DataRequired validator, 45, 109\n",
      "disabling CSRF tokens in unit tests, 226\n",
      "FlaskForm class, 44\n",
      "form fields, 45\n",
      "Length validator, 109\n",
      "login form, 109\n",
      "PasswordField class, 109\n",
      "rendering, 47\n",
      "StringField class, 45, 109\n",
      "SubmitField class, 45, 109\n",
      "using to render a form, 47\n",
      "validate_on_submit method, 49, 111\n",
      "validators, 46\n",
      "FlaskForm class, 44\n",
      "Flasky, Docker container image for, 260\n",
      "flasky.py, 266, 272\n",
      "coverage command, 222\n",
      "deploy command, 241, 253, 255, 260\n",
      "profile command, 239\n",
      "FLASKY_ADMIN environment variable, 83,\n",
      "131\n",
      "FLASKY_COMMENTS_PER_PAGE configu‐\n",
      "ration variable, 192\n",
      "FLASKY_POSTS_PER_PAGE configuration\n",
      "variable, 158\n",
      "FLASK_APP environment variable, 10, 66, 93,\n",
      "246\n",
      "setting by default, 97\n",
      "FLASK_CONFIG environment variable, 93,\n",
      "247\n",
      "FLASK_COVERAGE environment variable,\n",
      "223\n",
      "FLASK_DEBUG environment variable, 14, 93\n",
      "setting by default, 97\n",
      "followers, 171-187\n",
      "database relationships and, 171-178\n",
      "on the profile page, 178-180\n",
      "showing followed posts on home page,\n",
      "183-187\n",
      "for loops, 28\n",
      "foreign keys, 57, 64\n",
      "form.hidden_tag element, 47\n",
      "form.validate_on_submit method, 142\n",
      "format function, 40\n",
      "forms (see web forms)\n",
      "functools package, 133\n",
      "G\n",
      "g context variable, 18, 21\n",
      "GET request method (HTTP), 19\n",
      "in redirects, 51\n",
      "in RESTful APIs, 201\n",
      "resource handlers for blog posts, 213\n",
      "view function handling GET requests, 48\n",
      "get_flashed_messages function, 54\n",
      "Git\n",
      "downloading example code, 2\n",
      "server dedicated to Heroku application, 246\n",
      "source code for code examples, xiii\n",
      "uploading applications to Heroku server\n",
      "with git push, 254\n",
      "using with applications on Heroku, 244\n",
      "Gmail, Flask-Mail cofiguration for, 80\n",
      "Gravatar service, 146\n",
      "Gunicorn web server, 251-252\n",
      "H\n",
      "hashes\n",
      "MD5 hash for avatar URLs, 146\n",
      "caching of, 148\n",
      "password hashing, 102\n",
      "using Werkzeug security module, 102\n",
      "HEAD request method (HTTP), 19\n",
      "help from Flask developer community, 276\n",
      "Heroku platform, 244-255\n",
      "adding a Procfile, 252\n",
      "adding a top-level requirements file, 248\n",
      "addons:create command, 246\n",
      "CLI tool, 245\n",
      "config command, 246\n",
      "config:set command, 247\n",
      "configuring email, 248\n",
      "configuring logging, 247\n",
      "create command, 246\n",
      "284 \n",
      "| \n",
      "Index\n",
      "\n",
      "creating a Heroku account, 245\n",
      "creating an application, 245\n",
      "deploying application upgrades to, 255\n",
      "deploying applications to, using git push,\n",
      "254\n",
      "enabling secure HTTP with Flask-SSLify,\n",
      "249\n",
      "login command, 245\n",
      "logs ommand, 247\n",
      "maintenance command, 255\n",
      "provisioning a database, 246\n",
      "reviewing application logs, 255\n",
      "running a production web server, 251\n",
      "testing with heroku local command, 253\n",
      "hostname (database URLs), 61\n",
      "HTML\n",
      "Markdown-to-HTML converter, 164\n",
      "rendering of forms to, 47-48\n",
      "HTTP (secure), enabling with Flask-SSLify, 249\n",
      "HTTP authentication, 206\n",
      "HTTP methods, 19\n",
      "and resource handlers for RESTful web ser‐\n",
      "vice, 215\n",
      "request methods in RESTful APIs, 201\n",
      "HTTP status codes, 21\n",
      "404 error, 33, 145\n",
      "RESTful API error handler for 403 status\n",
      "code, 206\n",
      "returned by RESTful APIs, 204\n",
      "HTTPBasicAuth class, 207\n",
      "HTTPie, using to test web services, 217-218\n",
      "I\n",
      "images (container), 244\n",
      "building a Docker container image, 257\n",
      "cleaning up, 269\n",
      "importing files, template macros, 29\n",
      "included files, 29\n",
      "inheritance in Jinja2 templates, 29, 31\n",
      "init_app method, 88\n",
      "insert_roles method, 131\n",
      "integrated development environments (IDEs),\n",
      "275\n",
      "itsdangerous package, 119\n",
      "generating confirmation tokens for user\n",
      "accounts, 119\n",
      "token-based authentication support, 209\n",
      "J\n",
      "JavaScript files (Bootstrap), 30, 33\n",
      "JavaScript, moment.js library, 38\n",
      "Jinja2 package, 1, 26-30\n",
      "block directive, 29\n",
      "control structures, 28\n",
      "for directive, 28\n",
      "macro directive, 29\n",
      "extends directive, 30\n",
      "import directive, 29, 47\n",
      "include directive, 29\n",
      "rendering templates, 26\n",
      "safe filter, 28\n",
      "set directive, 195\n",
      "super macro, 30\n",
      "variables, 27\n",
      "filters for, 27\n",
      "wtf.quick_form macro, 110\n",
      "joins (database)\n",
      "joined lazy argument for back references,\n",
      "175\n",
      "using in databse query of followed posts,\n",
      "181\n",
      "jQuery.js library, 39\n",
      "JSON\n",
      "serializing resources to and from, 210-213\n",
      "use in RESTful web services, 202\n",
      "JSON Web Signatures (JWSs), 119\n",
      "jsonify helper function, 203\n",
      "junction tables (see association tables (data‐\n",
      "base))\n",
      "K\n",
      "key-value databases, 57\n",
      "Kubernetes, 270\n",
      "L\n",
      "language codes, 41\n",
      "links, 36\n",
      "blog post editor, 168\n",
      "moderate comments link in navigation bar,\n",
      "193\n",
      "permanent links to blog posts, 165-167\n",
      "profile edit link, 142\n",
      "profile edit link for administrator, 145\n",
      "to blog post comments, 192\n",
      "to user profile page in navigation bar, 140\n",
      "locale function, 41\n",
      "Index \n",
      "| \n",
      "285\n",
      "\n",
      "localization of dates and time, 38\n",
      "logging\n",
      "configuring on Heroku platform, 247\n",
      "Docker configuration for, 258\n",
      "docker logs command, 262\n",
      "of errors during production, 242\n",
      "reviewing application logs on Heroku plat‐\n",
      "form, 255\n",
      "setting up in traditional deployments, 272\n",
      "login view function, 111\n",
      "login_manager.anonymous_user attribute, 132\n",
      "login_manager.user_loader decorator, 108\n",
      "login_required decorator, 108, 114, 122, 208\n",
      "M\n",
      "macros, 29\n",
      "MAIL_PASSWORD environment variable, 80\n",
      "MAIL_USERNAME environment variable, 80\n",
      "make_response function, 21, 185\n",
      "many-to-many relationships, 65, 172\n",
      "advanced, 174\n",
      "followers helper methods, 176\n",
      "implementation as one-to-many rela‐\n",
      "tionships, 175\n",
      "self-referential, 174\n",
      "many-to-one relationships, 65\n",
      "Markdown, 162-165\n",
      "conversion to HTML on the server, 164\n",
      "post form enabled for, 162\n",
      "messages, flashing from forms, 53-55\n",
      "methods argument, 48\n",
      "microservices, 265\n",
      "Microsoft Windows (see Windows systems)\n",
      "migration scripts (database), 74\n",
      "creating with flask db migrate, 75\n",
      "model definition (database), 62-64\n",
      "moment.js library, 38\n",
      "formatting options for dates and time, 40\n",
      "MySQL databases, 264-265, 271\n",
      "N\n",
      "NameForm class, 45\n",
      "namespaces in blueprints, 92\n",
      "NoSQL databases, 57-58\n",
      "Flask support for, xi\n",
      "SQL databases vs., 59\n",
      "O\n",
      "OAuth2 authentication, 80\n",
      "object-document mappers (ODMs), 60\n",
      "object-relational mappers (ORMs), 60\n",
      "model, 62\n",
      "one-to-many relationships, 58, 64\n",
      "comments table to users table, 189\n",
      "many-to-many relationship implemented\n",
      "as, 175\n",
      "querying, 70\n",
      "one-to-one relationships, 65\n",
      "OPTIONS request method (HTTP), 19\n",
      "orchestration (container) with docker-\n",
      "compose, 265-269\n",
      "P\n",
      "PageDown library, 162\n",
      "pagination\n",
      "of large resource collections, 216\n",
      "of long blog post lists, 155-161\n",
      "adding a pagination widget, 158-161\n",
      "creating fake blog post data, 155\n",
      "rendering in pages, 157\n",
      "of user comments on posts, 192\n",
      "PasswordField class, 109\n",
      "passwords\n",
      "password in database URLs, 61\n",
      "security, 102-105\n",
      "updates and resets for user accounts, 125\n",
      "performance, 237-240\n",
      "logging slow database performance, 237-239\n",
      "source code profiling, 239-240\n",
      "permissions, 127\n",
      "comment moderation, 193\n",
      "custom decorators checking, 132\n",
      "evaluating for a user, 132\n",
      "in user roles database, 128\n",
      "constants for permissions, 128\n",
      "methods for managing, 129\n",
      "roles supported with permissions, 130\n",
      "unit tests for, 134\n",
      "permission_required decorator, 214\n",
      "pip, 5\n",
      "Platform as a Service (PaaS), 244\n",
      "POST request method (HTTP), 43\n",
      "in RESTful APIs, 201\n",
      "redirect response to POST requests, 51\n",
      "resource handler for blog posts, 214\n",
      "view function handling POST requests, 48\n",
      "286 \n",
      "| \n",
      "Index\n",
      "\n",
      "Post/Redirect/Get pattern, 52\n",
      "logins and, 112\n",
      "Postfix, 271\n",
      "Postgres databases, 246, 271\n",
      "presentation logic, 25\n",
      "primary key (database), 57\n",
      "primay key column, Flask-SQLAlchemy\n",
      "requirement for, 64\n",
      "Procfile (applications on Heroku), 252\n",
      "profile-header CSS class, 147\n",
      "profile-thumbnail CSS class, 147\n",
      "profiles, 137-149\n",
      "blog post author username and avatar, links\n",
      "to profile page, 153\n",
      "blog posts on profile pages, 154\n",
      "creating user profile page, 138-141\n",
      "editor for, 141-145\n",
      "administrator-level editor, 143-145\n",
      "user-level editor, 141\n",
      "followers on the profile page, 178-180\n",
      "information in, 137\n",
      "user avatars, 146-149\n",
      "profiling source code, 239\n",
      "proxy servers, 250, 271\n",
      "psycopg2 package, 248\n",
      "PUT request method (HTTP), 201\n",
      "PUT resource handler for blog posts, 215\n",
      "pymysql package, 264\n",
      "Python, 1\n",
      "creating virtual environments with Python\n",
      "2, 3\n",
      "creating virtual environments with Python\n",
      "3, 3\n",
      "database frameworks, 59-60\n",
      "database integration with Python shell, 72\n",
      "installing packages with pip, 5\n",
      "interpreter images on Docker Hub, 258\n",
      "Python Package Index, 276\n",
      "python-dotenv package, 272\n",
      "Q\n",
      "query object (database), 68\n",
      "R\n",
      "redirect function, 22, 53\n",
      "redirects, 22, 51-53\n",
      "SSL, 249\n",
      "Regexp form field validator, 116\n",
      "registration of new users, 115-118\n",
      "adding user registration form, 115\n",
      "registering users, 117\n",
      "regressions, 221\n",
      "relational databases, 57\n",
      "(see also SQL databases)\n",
      "Flask support for, xi\n",
      "relationships (database), 57, 64-65, 171-178\n",
      "dynamic relationships, 70\n",
      "many-to-many, 172\n",
      "advanced, 174\n",
      "querying a one-to-many relationship, 70\n",
      "self-referential, 174\n",
      "SQLAlchemy options for, 65\n",
      "relative URLs (in links), 37\n",
      "REMEMBER_COOKIE_DURATION option,\n",
      "111\n",
      "remote procedure call (RPC) protocols, 199\n",
      "rendering (templates), 25\n",
      "render_template function, 27, 49, 53\n",
      "Representational State Transfer (see REST)\n",
      "request context, 18\n",
      "request-response cycle, 17-22\n",
      "application and request contexts, 17\n",
      "HTTP request methods, 19\n",
      "request and response bodies in RESTful\n",
      "APIs, 201\n",
      "request dispatching, 18\n",
      "request hooks, 20\n",
      "request methods in RESTful APIs, 201\n",
      "request object, 19\n",
      "requests, 8\n",
      "response formats for RESTful API clients,\n",
      "205\n",
      "responses, 8, 21\n",
      "response object, 21\n",
      "requirements file, 93, 248\n",
      "development requirements file, 156\n",
      "top-level requirements file for Heroku plat‐\n",
      "form, 248\n",
      "resources\n",
      "implementing endpoints for, 213-216\n",
      "pagination of large collections, 216\n",
      "serializing to and from JSON, 210-213\n",
      "URLs for, 202\n",
      "resources for working with Flask, 275-277\n",
      "getting help, 276\n",
      "getting involved with the Flask community,\n",
      "277\n",
      "Index \n",
      "| \n",
      "287\n",
      "\n",
      "integrated development environments\n",
      "(IDEs), 275\n",
      "REST, 199\n",
      "defining characteristics for web services\n",
      "architecture, 199\n",
      "request and response bodies, 201\n",
      "request methods, 201\n",
      "resources, concept of, 200\n",
      "versioning of web services, 202\n",
      "RESTful web services with Flask, 203-218\n",
      "creating an API blueprint, 203\n",
      "error handling, 204\n",
      "implementing resource endpoints, 213-216\n",
      "pagination of large resource collections, 216\n",
      "serializing resources to and from JSON,\n",
      "210-213\n",
      "testing with HTTPie, 217-218\n",
      "token-based authentication, 208-210\n",
      "user authentication with Flask-HTTPAuth,\n",
      "206\n",
      "reverse proxy servers, 250, 271\n",
      "rich internet applications (RIAs), 199\n",
      "rich text posts, using Markdown and Flask-\n",
      "PageDown, 161-165\n",
      "handling rich text on the server, 164\n",
      "roles, 127-135\n",
      "adding to development database in shell ses‐\n",
      "sion, 134\n",
      "assignment of, 131\n",
      "database representation of, 127-131\n",
      "method creating roles, 130\n",
      "unit tests for, 134\n",
      "verification of, 132-135\n",
      "rolling back database sessions, 68\n",
      "Ronacher, Armin, 1\n",
      "route decorator, 133\n",
      "routes, 8\n",
      "access by authenticated users only, 108\n",
      "authentication token generation, 210\n",
      "blog post comments support, 191\n",
      "comment moderation, 196\n",
      "dynamic, 12\n",
      "editor for blog posts, 167\n",
      "follow route, 179\n",
      "home page route with blog posts, 152\n",
      "pagination support, 157\n",
      "in authentication blueprint, 105\n",
      "in blueprints, 91\n",
      "permanent links to blog posts, 165\n",
      "profile edit route, 141\n",
      "profile edit route for administrators, 144\n",
      "profile page route, 138\n",
      "profile page route with blog posts, 154\n",
      "registration route with confirmation email,\n",
      "120\n",
      "selection of all or followed posts, 184\n",
      "sign out, 113\n",
      "user registration, 117\n",
      "routing subsystem, 1\n",
      "rows (database), 57\n",
      "deleting, 68\n",
      "inserting, 66\n",
      "modifying, 68\n",
      "querying, 68\n",
      "S\n",
      "secret key, 44, 87, 119\n",
      "for applications on Heroku platform, 247\n",
      "secure HTTP, 249\n",
      "SelectField class, 144\n",
      "Selenium, end-to-end testing with, 230-234\n",
      "self-referential relationships, 174\n",
      "Sendmail, 271\n",
      "serialization\n",
      "deserializing resources from JSON, 212\n",
      "serializing resources to JSON, 210\n",
      "server setup in traditional deployment, 271\n",
      "server shutdown, 230\n",
      "session context variable, 18, 52\n",
      "session.get method, 53\n",
      "sessions (database), 67\n",
      "committing, 67\n",
      "rolling back, 68\n",
      "sessions (user), 44\n",
      "redirects and, 51\n",
      "set_cookie method, 21, 185\n",
      "shell context processor, 72\n",
      "SMTP server, 79\n",
      "smtplib package, 79\n",
      "source code profiling, 239\n",
      "SQL (Structured Query Language), 57\n",
      "SQL databases, 57\n",
      "NoSQL vs., 59\n",
      "SQLAlchemy, 60\n",
      "column options, 63\n",
      "column types, 63\n",
      "inspecting native SQL query generated by,\n",
      "69\n",
      "288 \n",
      "| \n",
      "Index\n",
      "\n",
      "Markdown text conversion to HTML, 164\n",
      "query executors, 69\n",
      "query filters, 69\n",
      "relationship options, 65\n",
      "SQLALCHEMY_DATABASE_URI, 61, 87\n",
      "SQLALCHEMY_TRACK_MODIFICATIONS,\n",
      "61\n",
      "SQLite databases, 271\n",
      "SSL_REDIRECT variable, 249, 250\n",
      "stack traces, 242\n",
      "stateless web services, 206\n",
      "static files, 37\n",
      "static methods, 131\n",
      "status codes (HTTP), 21\n",
      "StringField class, 45, 109\n",
      "SubmitField class, 45, 109\n",
      "super function, 30, 33\n",
      "syslog, 272\n",
      "T\n",
      "__tablename__ class variable, 62\n",
      "tables (database), 57\n",
      "creating, 66\n",
      "creating or upgrading in large application,\n",
      "96\n",
      "joins, 58\n",
      "teardown_request hook, 20\n",
      "templates, 25-41\n",
      "adding Permission class to template context,\n",
      "134\n",
      "Bootstrap integration using Flash-\n",
      "Bootstrap, 30-33\n",
      "comment moderation, 194\n",
      "confirmation email used by authentication\n",
      "blueprint, 121\n",
      "custom error pages, 33-36\n",
      "defined, 25\n",
      "edit blog post template, 167\n",
      "flash message rendering, 54\n",
      "Flask-PageDown template declaration, 162\n",
      "follower enhancements to profile header,\n",
      "178\n",
      "for authentication blueprint, 105\n",
      "for email messages, 81\n",
      "for login form, 109\n",
      "for new user registration form, 116\n",
      "for user profile, 139\n",
      "greeting logged-in user, 114\n",
      "home page template with blog posts, 152\n",
      "Jinja2 template engine, 26-30\n",
      "links, 36\n",
      "localizing dates and time with Flask-\n",
      "Moment, 38\n",
      "login template, updating to render login\n",
      "form, 112\n",
      "pagination footer for blog post lists, 160\n",
      "pagination template macro, 159\n",
      "permanent link to blog posts, 166\n",
      "profile page template with blog posts, 155\n",
      "static files, 37\n",
      "templates folder, 26\n",
      "using to render a form to HTML, 47\n",
      "test command to run unit tests, 95\n",
      "testing, 221-235\n",
      "assessing worth of, 234\n",
      "end-to-end, using Selenium, 230-234\n",
      "getting code coverage reports, 221-224\n",
      "of web services with HTTPie, 217-218\n",
      "password hashing tests, 104\n",
      "unit tests file for large applications, 94\n",
      "unit tests for roles and permissions, 134\n",
      "using Flask test client, 224-229\n",
      "testing web applications, 225-228\n",
      "testing web services, 228-229\n",
      "verifying login functionality, 114\n",
      "time (see dates and time)\n",
      "timestamps, working with, using Flask-\n",
      "Moment, 39\n",
      "token-based authentication, 208-210, 218\n",
      "transactions, 67\n",
      "(see also sessions (database))\n",
      "U\n",
      "unconfirmed accounts\n",
      "filtering in before_app_request handler, 122\n",
      "page presented asking for account confir‐\n",
      "mation, 123\n",
      "unittest package, 95\n",
      "URL fragments, 192\n",
      "URLs\n",
      "application routes, 9\n",
      "application URL map, 19\n",
      "avatar, 146\n",
      "database, in Flask-SQLAlchemy, 61\n",
      "for resources, 200\n",
      "for resoures\n",
      "fully-qualified, 202\n",
      "Index \n",
      "| \n",
      "289\n",
      "\n",
      "in confirmation emails for user accounts,\n",
      "121\n",
      "in links, 37\n",
      "url_for function, 37, 53, 92, 121\n",
      "url_prefix argument, 106\n",
      "user authentication (see authentication)\n",
      "user comments, 189-197\n",
      "database representation of, 189-191\n",
      "moderation of, 193-197\n",
      "submission and display of, 191-193\n",
      "user loader function, 108\n",
      "User model\n",
      "preparing for logins, 107\n",
      "preparing for password hashing, 102\n",
      "user account token generation and verifica‐\n",
      "tion, 119\n",
      "user profiles (see profiles)\n",
      "user roles (see roles)\n",
      "user sessions, 44, 52\n",
      "expiration, long-term cookie for, 111\n",
      "User.can method, 153\n",
      "username (database URLs), 61\n",
      "users\n",
      "making existing users their own followers,\n",
      "186\n",
      "making their own followers on construc‐\n",
      "tion, 186\n",
      "UTC (Coordinated Universal Time), 38\n",
      "uWSGI web server, 252\n",
      "V\n",
      "validate_on_submit method, 49, 111\n",
      "ValidationError, 116, 212\n",
      "RESTful API error handler for, 213\n",
      "validators, 44, 109\n",
      "built-in, WTForms package, 46\n",
      "implemented as methods, 116\n",
      "variables, 26-27\n",
      "filters for, 27\n",
      "venv package (Python), 3\n",
      "verify_password method, 111\n",
      "view functions, 8\n",
      "confirming user accounts, 121\n",
      "database use in, 71-72\n",
      "for blog post comments, 191\n",
      "for blog post editor, 168\n",
      "for followers on profile page, 179\n",
      "for permanent links to blog posts, 165\n",
      "form handling in, 48-51\n",
      "in authentication blueprint, 105\n",
      "in blueprints, 92\n",
      "login function implementation, 111\n",
      "order of decorators in, 133\n",
      "purposes of, 25\n",
      "virtual environments, 2\n",
      "activating, 4\n",
      "creating with Python 2, 3\n",
      "creating with Python 3, 3\n",
      "deactivating, 5\n",
      "installing Flask into, 5\n",
      "using without activating, 5\n",
      "virtual machines, 256\n",
      "virtual servers, 243\n",
      "virtualenv utility, 3\n",
      "volumes (Docker), removing, 269\n",
      "W\n",
      "Waitress web server, 252\n",
      "web browsers, Selenium automation tool, 230\n",
      "web dynos (Heroku), 244\n",
      "web forms, 43-55\n",
      "blog post form, 151\n",
      "configuration, 44\n",
      "flashing a message, 53-55\n",
      "form classes, 44-47\n",
      "generated by Flash-WTF, CSRF tokens in,\n",
      "226\n",
      "handling in view functions, 48-51\n",
      "HTML rendering of, 47-48\n",
      "in blueprints, 92\n",
      "login form, 109\n",
      "profile edit form, 141\n",
      "profile editing form for administrators, 143\n",
      "redirects and user sessions, 51-53\n",
      "user registration form, 115\n",
      "Web Server Gateway Interface (WSGI), 1, 7\n",
      "web servers\n",
      "installing in traditional deployment, 271\n",
      "running production web server on Heroku,\n",
      "251\n",
      "web services, 199, 203\n",
      "(see also RESTful web services with Flask)\n",
      "testing RESTful web services using Flask test\n",
      "client, 228-229\n",
      "Werkzeug (security module), 1, 102, 242\n",
      "generate_password_hash function, 103\n",
      "ProfilerMiddleware WSGI middleware, 240\n",
      "ProxyFix WSGI middleware, 250\n",
      "290 \n",
      "| \n",
      "Index\n",
      "\n",
      "verify_password method, 103\n",
      "Windows Subsystem for Linux (WSL), 2\n",
      "Windows systems, 2\n",
      "Docker for Windows, Hyper-V feature and,\n",
      "257\n",
      "testing Heroku deployment, using Waitress\n",
      "web server, 252\n",
      "worker dynos (Heroku), 244\n",
      "wtf.quick_form function, 47\n",
      "WTForms package, 43\n",
      "built-in validators, 46\n",
      "Regexp validator, 116\n",
      "SelectField wrapper class, 144\n",
      "standard HTML fields supported by, 45\n",
      "X\n",
      "XML in RESTful web services, 202\n",
      "Index \n",
      "| \n",
      "291\n",
      "\n",
      "About the Author\n",
      "Miguel Grinberg has over 25 years of experience as a software engineer. He has a\n",
      "blog where he writes about a variety of topics, including web development, robotics,\n",
      "photography, and the occasional movie review. He lives in Portland, Oregon.\n",
      "Colophon\n",
      "The animal on the cover of Flask Web Development is a Pyrenean Mastiff (a breed of\n",
      "Canis lupus familiaris). These giant Spanish dogs are descended from an ancient live‐\n",
      "stock guardian dog called the Molossus, which was bred by the Greeks and Romans\n",
      "and is now extinct. However, this ancestor is known to have played a role in the cre‐\n",
      "ation of many breeds that are common today, such as the Rottweiler, Great Dane,\n",
      "Newfoundland, and Cane Corso. Pyrenean Mastiffs have only been recognized as a\n",
      "pure breed since 1977, and the Pyrenean Mastiff Club of America is working to pro‐\n",
      "mote these dogs as pets in the United States.\n",
      "After the Spanish Civil War, the population of Pyrenean Mastiffs in their native\n",
      "homeland plummeted, and the breed only survived due to the dedicated work of a\n",
      "few scattered breeders throughout the country. The modern gene pool for Pyreneans\n",
      "stems from this postwar population, making them prone to genetic diseases like hip\n",
      "dysplasia. Today, responsible owners make sure their dogs are tested for diseases and\n",
      "x-rayed to look for hip abnormalities before being bred.\n",
      "Adult male Pyrenean Mastiffs can reach upwards of 200 pounds when fully grown, so\n",
      "owning this dog requires a commitment to good training and plenty of outdoor time.\n",
      "Despite their size and history as hunters of bears and wolves, the Pyrenean has a very\n",
      "calm temperament and is an excellent family dog. They can be relied upon to take\n",
      "care of children and protect the home, while at the same time being docile with other\n",
      "dogs. With proper socialization and strong leadership, the Pyrenean Mastiff thrives in\n",
      "a home environment and will provide an excellent guardian and companion.\n",
      "Many of the animals on O’Reilly covers are endangered; all of them are important to\n",
      "the world. To learn more about how you can help, go to animals.oreilly.com.\n",
      "The cover image is from J. G. Wood’s Animate Creation. The cover fonts are URW\n",
      "Typewriter and Guardian Sans. The text font is Adobe Minion Pro; the heading font\n",
      "is Adobe Myriad Condensed; and the code font is Dalton Maag’s Ubuntu Mono.\n",
      "\n",
      "Text Statistics:\n",
      "Word Count: 98815\n",
      "Sentence Count: 4310\n",
      "Average Word Length: 4.45\n"
     ]
    }
   ],
   "source": [
    "import fitz  # PyMuPDF\n",
    "import nltk\n",
    "from nltk.tokenize import word_tokenize, sent_tokenize\n",
    "\n",
    "# Download necessary NLTK data files\n",
    "nltk.download('punkt')\n",
    "\n",
    "def extract_text_from_pdf(pdf_path):\n",
    "    pdf_document = fitz.open(pdf_path)\n",
    "    text_content = []\n",
    "\n",
    "    for page_num in range(pdf_document.page_count):\n",
    "        page = pdf_document.load_page(page_num)\n",
    "        text_content.append(page.get_text())\n",
    "\n",
    "    return '\\n'.join(text_content)\n",
    "\n",
    "def text_statistics(text):\n",
    "    words = word_tokenize(text)\n",
    "    sentences = sent_tokenize(text)\n",
    "    word_count = len(words)\n",
    "    sentence_count = len(sentences)\n",
    "    average_word_length = sum(len(word) for word in words) / word_count\n",
    "\n",
    "    return {\n",
    "        'word_count': word_count,\n",
    "        'sentence_count': sentence_count,\n",
    "        'average_word_length': average_word_length\n",
    "    }\n",
    "\n",
    "# Example usage\n",
    "fname = '/home/ys/Documents/Flask Web Development - Miguel Grinberg.pdf'\n",
    "text = extract_text_from_pdf(fname)\n",
    "print(text)\n",
    "\n",
    "# Generate text statistics\n",
    "stats = text_statistics(text)\n",
    "print(\"Text Statistics:\")\n",
    "print(f\"Word Count: {stats['word_count']}\")\n",
    "print(f\"Sentence Count: {stats['sentence_count']}\")\n",
    "print(f\"Average Word Length: {stats['average_word_length']:.2f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "General",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}