App Description

I built a computer vision app to recognize 12 kinds of beer using fastai. This is project is based on chapter 2 of Jeremy Howard's Deep Learning book and lesson 2 of the Deep Learning course v4.

The app) is now running on heroku and works best on mobile phones. I'm using the free heroku service so it can take 30 seconds or so to load.

Collecting Data

The first step is downloading Images from Bing with Azure API. I signed up for a Microsoft Azure account to get an API key to search Bing.

from fastbook import *
from fastai.vision.widgets import *
import yaml

key = yaml.safe_load(open('key.yaml'))['AZURE_SEARCH_KEY']

I picked 12 beers for my app to recognize

beer_types = 'toppling goliath pompeii','toppling goliath pseudo sue','surly furious', \
    'busch light', 'bells two hearted', 'toppling goliath king sue', \
    'founders kentucky breakfast stout', 'unibroue la fin du monde', 'bells hopslam', \
    'dogfish head 60 minute ipa', 'founders solid gold', 'founders all day ipa'

I looped through the 12 beers and used fastai's search_images_bing function to download 150 of each type of beer

path = Path('beers')
if not path.exists():
    path.mkdir()
    for o in beer_types:
        dest = (path/o)
        dest.mkdir(exist_ok=True)
        results = search_images_bing(key, f'{o} beer')
        download_images(dest, urls=results.attrgot('contentUrl'))

Next, I checked to see if any of my image downloads failed and removed them from the dataset.

fns = get_image_files(path)
fns

failed = verify_images(fns)
failed

(#0) []

failed.map(Path.unlink);

Loading the data into a fastai datablock

beers = DataBlock(
    blocks=(ImageBlock, CategoryBlock), 
    get_items=get_image_files, 
    splitter=RandomSplitter(valid_pct=0.2, seed=42),
    get_y=parent_label,
    item_tfms=Resize(128))

dls = beers.dataloaders(path)

dls.valid.show_batch(max_n=4, nrows=1)

beers = beers.new(item_tfms=Resize(128, ResizeMethod.Squish))
dls = beers.dataloaders(path)
dls.valid.show_batch(max_n=4, nrows=1)

bears = bears.new(item_tfms=Resize(128, ResizeMethod.Pad, pad_mode='zeros'))
dls = bears.dataloaders(path)
dls.valid.show_batch(max_n=4, nrows=1)

bears = bears.new(item_tfms=RandomResizedCrop(128, min_scale=0.3))
dls = bears.dataloaders(path)
dls.train.show_batch(max_n=4, nrows=1, unique=True)

Data Augmentation

beers = beers.new(item_tfms=Resize(128), batch_tfms=aug_transforms(mult=2))
dls = beers.dataloaders(path)
dls.train.show_batch(max_n=8, nrows=2, unique=True)

Training Your Model, and Using It to Clean Your Data

beers = beers.new(
    item_tfms=RandomResizedCrop(224, min_scale=0.5),
    batch_tfms=aug_transforms())
dls = beers.dataloaders(path)

learn = cnn_learner(dls, resnet18, metrics=error_rate)
learn.fine_tune(4)

interp = ClassificationInterpretation.from_learner(learn)
interp.plot_confusion_matrix(figsize=[10,10])

cleaner = ImageClassifierCleaner(learn)
cleaner

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
<ipython-input-15-1df86057f98d> in <module>
----> 1 cleaner = ImageClassifierCleaner(learn)
      2 cleaner

NameError: name 'learn' is not defined

for idx in cleaner.delete(): cleaner.fns[idx].unlink()
for idx,cat in cleaner.change(): shutil.move(str(cleaner.fns[idx]), path/cat)

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
<ipython-input-61-bb2fe04ddcb9> in <module>
----> 1 for idx in cleaner.delete(): cleaner.fns[idx].unlink()
      2 for idx,cat in cleaner.change(): shutil.move(str(cleaner.fns[idx]), path/cat)

/opt/conda/envs/fastai/lib/python3.8/pathlib.py in unlink(self, missing_ok)
   1322             self._raise_closed()
   1323         try:
-> 1324             self._accessor.unlink(self)
   1325         except FileNotFoundError:
   1326             if not missing_ok:

FileNotFoundError: [Errno 2] No such file or directory: 'beers/founders solid gold/00000065.jpg'

interp.plot_top_losses(5, nrows=1)

cleaner = ImageClassifierCleaner(learn)
cleaner

Turning Your Model into an Online Application

Using the Model for Inference

learn.export()

path = Path()
path.ls(file_exts='.pkl')

(#1) [Path('export.pkl')]

learn_inf = load_learner(path/'export.pkl')

learn_inf.predict('images/grizzly.jpg')

('toppling goliath pseudo sue',
 tensor(6),
 tensor([2.2459e-03, 4.9390e-03, 1.4955e-02, 4.9468e-04, 8.0316e-03, 2.5601e-02, 9.4373e-01]))

learn_inf.dls.vocab

['bells two hearted', 'busch light', 'founders kentucky breakfast stout', 'surly furious', 'toppling goliath king sue', 'toppling goliath pompeii', 'toppling goliath pseudo sue']

Creating a Notebook App from the Model

btn_upload = widgets.FileUpload()
btn_upload

img = PILImage.create(btn_upload.data[-1])

out_pl = widgets.Output()
out_pl.clear_output()
with out_pl: display(img.to_thumb(128,128))
out_pl

pred,pred_idx,probs = learn_inf.predict(img)

lbl_pred = widgets.Label()
lbl_pred.value = f'Prediction: {pred.title()}; Probability: {100*probs[pred_idx]:.01f}%'
lbl_pred

btn_run = widgets.Button(description='Classify')
btn_run

def on_click_classify(change):
    img = PILImage.create(btn_upload.data[-1])
    out_pl.clear_output()
    with out_pl: display(img.to_thumb(128,128))
    pred,pred_idx,probs = learn_inf.predict(img)
    lbl_pred.value = f'Prediction: {pred}; Probability: {probs[pred_idx]:.04f}'

btn_run.on_click(on_click_classify)

VBox([widgets.Label('Select your bear!'), 
      btn_upload, btn_run, out_pl, lbl_pred])

epoch	train_loss	valid_loss	error_rate	time
0	0.866057	0.368824	0.099617	00:18
1	0.597279	0.193733	0.042146	00:19
2	0.417497	0.175624	0.045977	00:18
3	0.322531	0.175689	0.042146	00:19

Beer Detector