metadata-crawler
This is a simple node.js module for scraping meta information from web pages.
At the moment this module supports:
- the
<title>tag in the document head - meta[name="description"] tag
- link[rel="canonical"] tag
- img elements on the page
- many Open Graph tags (such as: title, description, url, type, image, video)
Install
npm install metatag-crawler --save
Usage
var scrape = require('metatag-crawler');
scrape('https://www.youtube.com/watch?v=jNQXAC9IVRw', function(err, data) {});
// if you do not want the relative URLs to be resolved
scrape('https://www.youtube.com/watch?v=jNQXAC9IVRw', { resolveUrls: false }, function(err, data) {});
Example
var scrape = require('metatag-crawler');
scrape('https://www.youtube.com/watch?v=jNQXAC9IVRw', function(err, data) {
console.log(err); // null
console.log(data.meta.title); // Me at the zoo
console.log(data.meta.description); // The first video on YouTube. Maybe it's time to go back to the zoo?
console.log(data.meta.canonical); // https://www.youtube.com/watch?v=jNQXAC9IVRw
console.log(data);
/*
{
"meta": {
"title": "Me at the zoo - YouTube",
"description": "The first video on YouTube. Maybe it's time to go back to the zoo?",
"canonical": "https://www.youtube.com/watch?v=jNQXAC9IVRw"
},
"images": [
{
"url": "https://s.ytimg.com/yts/img/pixel-vfl3z5WfW.gif"
},
{
"url": "https://s.ytimg.com/yts/img/pixel-vfl3z5WfW.gif",
"width": 48,
"height": 48
},
{
"url": "https://s.ytimg.com/yts/img/pixel-vfl3z5WfW.gif",
"width": 120,
"height": 90
},
{
"url": "https://s.ytimg.com/yts/img/pixel-vfl3z5WfW.gif",
"width": 120,
"height": 90
},
{
"url": "https://s.ytimg.com/yts/img/pixel-vfl3z5WfW.gif",
"width": 120,
"height": 90
},
{
"url": "https://s.ytimg.com/yts/img/pixel-vfl3z5WfW.gif",
"width": 120,
"height": 90
},
{
"url": "https://s.ytimg.com/yts/img/pixel-vfl3z5WfW.gif",
"width": 120,
"height": 90
},
{
"url": "https://s.ytimg.com/yts/img/pixel-vfl3z5WfW.gif",
"width": 120,
"height": 90
},
{
"url": "https://s.ytimg.com/yts/img/pixel-vfl3z5WfW.gif",
"width": 120,
"height": 90
},
{
"url": "https://s.ytimg.com/yts/img/pixel-vfl3z5WfW.gif",
"width": 120,
"height": 90
},
{
"url": "https://s.ytimg.com/yts/img/pixel-vfl3z5WfW.gif",
"width": 120,
"height": 90
},
{
"url": "https://s.ytimg.com/yts/img/pixel-vfl3z5WfW.gif",
"width": 120,
"height": 90
},
{
"url": "https://s.ytimg.com/yts/img/pixel-vfl3z5WfW.gif",
"width": 120,
"height": 90
},
{
"url": "https://s.ytimg.com/yts/img/pixel-vfl3z5WfW.gif",
"width": 120,
"height": 90
},
{
"url": "https://s.ytimg.com/yts/img/pixel-vfl3z5WfW.gif",
"width": 120,
"height": 90
},
{
"url": "https://s.ytimg.com/yts/img/pixel-vfl3z5WfW.gif",
"width": 120,
"height": 90
},
{
"url": "https://s.ytimg.com/yts/img/pixel-vfl3z5WfW.gif",
"width": 120,
"height": 90
},
{
"url": "https://s.ytimg.com/yts/img/pixel-vfl3z5WfW.gif",
"width": 120,
"height": 90
},
{
"url": "https://s.ytimg.com/yts/img/pixel-vfl3z5WfW.gif",
"width": 120,
"height": 90
},
{
"url": "https://s.ytimg.com/yts/img/pixel-vfl3z5WfW.gif",
"width": 120,
"height": 90
},
{
"url": "https://s.ytimg.com/yts/img/pixel-vfl3z5WfW.gif",
"width": 120,
"height": 90
},
{
"url": "https://s.ytimg.com/yts/img/pixel-vfl3z5WfW.gif",
"width": 120,
"height": 90
}
],
"og": {
"title": "Me at the zoo",
"description": "The first video on YouTube. Maybe it's time to go back to the zoo?",
"url": "https://www.youtube.com/watch?v=jNQXAC9IVRw",
"site_name": "YouTube",
"type": "video",
"images": [
{
"url": "https://i.ytimg.com/vi/jNQXAC9IVRw/hqdefault.jpg"
}
],
"videos": [
{
"url": "https://www.youtube.com/embed/jNQXAC9IVRw",
"secure_url": "https://www.youtube.com/embed/jNQXAC9IVRw",
"type": "text/html",
"width": 480,
"height": 360
},
{
"url": "http://www.youtube.com/v/jNQXAC9IVRw?version=3&autohide=1",
"secure_url": "https://www.youtube.com/v/jNQXAC9IVRw?version=3&autohide=1",
"type": "application/x-shockwave-flash",
"width": 480,
"height": 360
}
]
}
}
*/
});Changes in version 2.x
In version 2 the module does not automatically merge properties from different meta information sources.
Earlier we used to the the following: title = opengpraph.title || meta.title and so on with description and canonical url.
This is no longer the case, we provide all information available on the page. If you need the old behavior, please process the result like this:
scrape('https://www.youtube.com/watch?v=jNQXAC9IVRw', function(err, data) {
var oldStyleData = {
title: data.og.title || data.meta.title,
description: data.og.descriptions || data.meta.description,
images: og.images,
videos: og.videos
};
// ... do what you used to do before with the oldStyleData
});
React
Typescript
Django
Laravel
Facebook
Microsoft
Google
Alibaba
D3
Tencent