summaryrefslogtreecommitdiff
blob: 6acf34db258d4182bc8afcd754d0e19eeb898c17 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
<?php
/**
 * Class with methods to extract metadata from a post/page about videos, images, links, mentions embedded
 * in or attached to the post/page.
 *
 * @todo Additionally, have some filters on number of items in each field
 */
class Jetpack_Media_Meta_Extractor {

	// Some consts for what to extract
	const ALL = 255;
	const LINKS = 1;
	const MENTIONS = 2;
	const IMAGES = 4;
	const SHORTCODES = 8; // Only the keeper shortcodes below
	const EMBEDS = 16;
	const HASHTAGS = 32;

	// For these, we try to extract some data from the shortcode, rather than just recording its presence (which we do for all)
	// There should be a function get_{shortcode}_id( $atts ) or static method SomethingShortcode::get_{shortcode}_id( $atts ) for these.
	private static $KEEPER_SHORTCODES = array(
		'youtube',
		'vimeo',
		'hulu',
		'ted',
		'wpvideo',
		'videopress',
	);

	/**
	 * Gets the specified media and meta info from the given post.
	 * NOTE: If you have the post's HTML content already and don't need image data, use extract_from_content() instead.
	 *
	 * @param $blog_id The ID of the blog
	 * @param $post_id The ID of the post
	 * @param $what_to_extract (int) A mask of things to extract, e.g. Jetpack_Media_Meta_Extractor::IMAGES | Jetpack_Media_Meta_Extractor::MENTIONS
	 * @returns a structure containing metadata about the embedded things, or empty array if nothing found, or WP_Error on error
	 */
	static public function extract( $blog_id, $post_id, $what_to_extract = self::ALL ) {

		// multisite?
		if ( function_exists( 'switch_to_blog') )
			switch_to_blog( $blog_id );

		$post = get_post( $post_id );
		$content = $post->post_title . "\n\n" . $post->post_content;
		$char_cnt = strlen( $content );

		//prevent running extraction on really huge amounts of content
		if ( $char_cnt > 100000 ) //about 20k English words
			$content = substr( $content, 0, 100000 );

		$extracted = array();

		// Get images first, we need the full post for that
		if ( self::IMAGES & $what_to_extract ) {
			$extracted = self::get_image_fields( $post );

			// Turn off images so we can safely call extract_from_content() below
			$what_to_extract = $what_to_extract - self::IMAGES;
		}

		if ( function_exists( 'switch_to_blog') )
			restore_current_blog();

		// All of the other things besides images can be extracted from just the content
		$extracted = self::extract_from_content( $content, $what_to_extract, $extracted );

		return $extracted;
	}

	/**
	 * Gets the specified meta info from the given post content.
	 * NOTE: If you want IMAGES, call extract( $blog_id, $post_id, ...) which will give you more/better image extraction
	 * This method will give you an error if you ask for IMAGES.
	 *
	 * @param $content The HTML post_content of a post
	 * @param $what_to_extract (int) A mask of things to extract, e.g. Jetpack_Media_Meta_Extractor::IMAGES | Jetpack_Media_Meta_Extractor::MENTIONS
	 * @param $already_extracted (array) Previously extracted things, e.g. images from extract(), which can be used for x-referencing here
	 * @returns a structure containing metadata about the embedded things, or empty array if nothing found, or WP_Error on error
	 */
	static public function extract_from_content( $content, $what_to_extract = self::ALL, $already_extracted = array() ) {
		$stripped_content = self::get_stripped_content( $content );

		// Maybe start with some previously extracted things (e.g. images from extract()
		$extracted = $already_extracted;

		// Embedded media objects will have already been converted to shortcodes by pre_kses hooks on save.

 		if ( self::IMAGES & $what_to_extract ) {
			$images = Jetpack_Media_Meta_Extractor::extract_images_from_content( $stripped_content, array() );
			$extracted = array_merge( $extracted, $images );
		}

		// ----------------------------------- MENTIONS ------------------------------

		if ( self::MENTIONS & $what_to_extract ) {
			if ( preg_match_all( '/(^|\s)@(\w+)/u', $stripped_content, $matches ) ) {
				$mentions = array_values( array_unique( $matches[2] ) ); //array_unique() retains the keys!
				$mentions = array_map( 'strtolower', $mentions );
				$extracted['mention'] = array( 'name' => $mentions );
				if ( !isset( $extracted['has'] ) )
					$extracted['has'] = array();
				$extracted['has']['mention'] = count( $mentions );
			}
		}

		// ----------------------------------- HASHTAGS ------------------------------
		/** Some hosts may not compile with --enable-unicode-properties and kick a warning:
		  *   Warning: preg_match_all() [function.preg-match-all]: Compilation failed: support for \P, \p, and \X has not been compiled
		  * Therefore, we only run this code block on wpcom, not in Jetpack.
		 */
		if ( ( defined( 'IS_WPCOM' ) && IS_WPCOM ) && ( self::HASHTAGS & $what_to_extract ) ) {
			//This regex does not exactly match Twitter's
			// if there are problems/complaints we should implement this:
			//   https://github.com/twitter/twitter-text/blob/master/java/src/com/twitter/Regex.java
			if ( preg_match_all( '/(?:^|\s)#(\w*\p{L}+\w*)/u', $stripped_content, $matches ) ) {
				$hashtags = array_values( array_unique( $matches[1] ) ); //array_unique() retains the keys!
				$hashtags = array_map( 'strtolower', $hashtags );
				$extracted['hashtag'] = array( 'name' => $hashtags );
				if ( !isset( $extracted['has'] ) )
					$extracted['has'] = array();
				$extracted['has']['hashtag'] = count( $hashtags );
			}
		}

		// ----------------------------------- SHORTCODES ------------------------------

		// Always look for shortcodes.
		// If we don't want them, we'll just remove them, so we don't grab them as links below
		$shortcode_pattern = '/' . get_shortcode_regex() . '/s';
 		if ( preg_match_all( $shortcode_pattern, $content, $matches ) ) {

			$shortcode_total_count = 0;
			$shortcode_type_counts = array();
			$shortcode_types = array();
			$shortcode_details = array();

			if ( self::SHORTCODES & $what_to_extract ) {

				foreach( $matches[2] as $key => $shortcode ) {
					//Elasticsearch (and probably other things) doesn't deal well with some chars as key names
					$shortcode_name = preg_replace( '/[.,*"\'\/\\\\#+ ]/', '_', $shortcode );

					$attr = shortcode_parse_atts( $matches[3][ $key ] );

					$shortcode_total_count++;
					if ( ! isset( $shortcode_type_counts[$shortcode_name] ) )
						$shortcode_type_counts[$shortcode_name] = 0;
					$shortcode_type_counts[$shortcode_name]++;

					// Store (uniquely) presence of all shortcode regardless of whether it's a keeper (for those, get ID below)
					// @todo Store number of occurrences?
					if ( ! in_array( $shortcode_name, $shortcode_types ) )
						$shortcode_types[] = $shortcode_name;

					// For keeper shortcodes, also store the id/url of the object (e.g. youtube video, TED talk, etc.)
					if ( in_array( $shortcode, self::$KEEPER_SHORTCODES ) ) {
						unset( $id ); // Clear shortcode ID data left from the last shortcode
						// We'll try to get the salient ID from the function jetpack_shortcode_get_xyz_id()
						// If the shortcode is a class, we'll call XyzShortcode::get_xyz_id()
						$shortcode_get_id_func = "jetpack_shortcode_get_{$shortcode}_id";
						$shortcode_class_name = ucfirst( $shortcode ) . 'Shortcode';
						$shortcode_get_id_method = "get_{$shortcode}_id";
						if ( function_exists( $shortcode_get_id_func ) ) {
							$id = call_user_func( $shortcode_get_id_func, $attr );
						} else if ( method_exists( $shortcode_class_name, $shortcode_get_id_method ) ) {
							$id = call_user_func( array( $shortcode_class_name, $shortcode_get_id_method ), $attr );
						}
						if ( ! empty( $id )
							&& ( ! isset( $shortcode_details[$shortcode_name] ) || ! in_array( $id, $shortcode_details[$shortcode_name] ) ) )
							$shortcode_details[$shortcode_name][] = $id;
					}
				}

				if ( $shortcode_total_count > 0 ) {
					// Add the shortcode info to the $extracted array
					if ( !isset( $extracted['has'] ) )
						$extracted['has'] = array();
					$extracted['has']['shortcode'] = $shortcode_total_count;
					$extracted['shortcode'] = array();
					foreach ( $shortcode_type_counts as $type => $count )
						$extracted['shortcode'][$type] = array( 'count' => $count );
					if ( ! empty( $shortcode_types ) )
						$extracted['shortcode_types'] = $shortcode_types;
					foreach ( $shortcode_details as $type => $id )
						$extracted['shortcode'][$type]['id'] = $id;
				}
			}

			// Remove the shortcodes form our copy of $content, so we don't count links in them as links below.
			$content = preg_replace( $shortcode_pattern, ' ', $content );
		}

		// ----------------------------------- LINKS ------------------------------

		if ( self::LINKS & $what_to_extract ) {

			// To hold the extracted stuff we find
			$links = array();

			// @todo Get the text inside the links?

			// Grab any links, whether in <a href="..." or not, but subtract those from shortcodes and images
			// (we treat embed links as just another link)
			if ( preg_match_all( '#(?:^|\s|"|\')(https?://([^\s()<>]+(?:\([\w\d]+\)|([^[:punct:]\s]|/))))#', $content, $matches ) ) {

				foreach ( $matches[1] as $link_raw ) {
					$url = parse_url( $link_raw );

					// Data URI links
					if ( isset( $url['scheme'] ) && 'data' === $url['scheme'] )
						continue;

					// Remove large (and likely invalid) links
					if ( 4096 < strlen( $link_raw ) )
						continue;

					// Build a simple form of the URL so we can compare it to ones we found in IMAGES or SHORTCODES and exclude those
					$simple_url = $url['scheme'] . '://' . $url['host'] . ( ! empty( $url['path'] ) ? $url['path'] : '' );
					if ( isset( $extracted['image']['url'] ) ) {
						if ( in_array( $simple_url, (array) $extracted['image']['url'] ) )
							continue;
					}

					list( $proto, $link_all_but_proto ) = explode( '://', $link_raw );

					// Build a reversed hostname
					$host_parts = array_reverse( explode( '.', $url['host'] ) );
					$host_reversed = '';
					foreach ( $host_parts as $part ) {
						$host_reversed .= ( ! empty( $host_reversed ) ? '.' : '' ) . $part;
					}

					$link_analyzed = '';
					if ( !empty( $url['path'] ) ) {
						// The whole path (no query args or fragments)
						$path = substr( $url['path'], 1 ); // strip the leading '/'
						$link_analyzed .= ( ! empty( $link_analyzed ) ? ' ' : '' ) . $path;

						// The path split by /
						$path_split = explode( '/', $path );
						if ( count( $path_split ) > 1 ) {
							$link_analyzed .= ' ' . implode( ' ', $path_split );
						}

						// The fragment
						if ( ! empty( $url['fragment'] ) )
							$link_analyzed .= ( ! empty( $link_analyzed ) ? ' ' : '' ) . $url['fragment'];
					}

					// @todo Check unique before adding
					$links[] = array(
						'url' => $link_all_but_proto,
						'host_reversed' => $host_reversed,
						'host' => $url['host'],
					);
				}

			}

			$link_count = count( $links );
			if ( $link_count ) {
				$extracted[ 'link' ] = $links;
				if ( !isset( $extracted['has'] ) )
					$extracted['has'] = array();
				$extracted['has']['link'] = $link_count;
			}
		}

		// ----------------------------------- EMBEDS ------------------------------

		//Embeds are just individual links on their own line
		if ( self::EMBEDS & $what_to_extract ) {

			if ( !function_exists( '_wp_oembed_get_object' ) )
				include( ABSPATH . WPINC . '/class-oembed.php' );

			// get an oembed object
			$oembed = _wp_oembed_get_object();

			// Grab any links on their own lines that may be embeds
			if ( preg_match_all( '|^\s*(https?://[^\s"]+)\s*$|im', $content, $matches ) ) {

				// To hold the extracted stuff we find
				$embeds = array();

				foreach ( $matches[1] as $link_raw ) {
					$url = parse_url( $link_raw );

					list( $proto, $link_all_but_proto ) = explode( '://', $link_raw );

					// Check whether this "link" is really an embed.
					foreach ( $oembed->providers as $matchmask => $data ) {
						list( $providerurl, $regex ) = $data;

						// Turn the asterisk-type provider URLs into regex
						if ( !$regex ) {
							$matchmask = '#' . str_replace( '___wildcard___', '(.+)', preg_quote( str_replace( '*', '___wildcard___', $matchmask ), '#' ) ) . '#i';
							$matchmask = preg_replace( '|^#http\\\://|', '#https?\://', $matchmask );
						}

						if ( preg_match( $matchmask, $link_raw ) ) {
							$provider = str_replace( '{format}', 'json', $providerurl ); // JSON is easier to deal with than XML
							$embeds[] = $link_all_but_proto; // @todo Check unique before adding

							// @todo Try to get ID's for the ones we care about (shortcode_keepers)
							break;
						}
					}
				}

				if ( ! empty( $embeds ) ) {
					if ( !isset( $extracted['has'] ) )
						$extracted['has'] = array();
					$extracted['has']['embed'] = count( $embeds );
					$extracted['embed'] = array( 'url' => array() );
					foreach ( $embeds as $e )
						$extracted['embed']['url'][] = $e;
				}
			}
		}

		return $extracted;
	}

	/**
	 * @param $post A post object
	 * @param $args (array) Optional args, see defaults list for details
	 * @returns array Returns an array of all images meeting the specified criteria in $args
	 *
	 * Uses Jetpack Post Images
	 */
	private static function get_image_fields( $post, $args = array() ) {

		$defaults = array(
			'width'               => 200, // Required minimum width (if possible to determine)
			'height'              => 200, // Required minimum height (if possible to determine)
		);

		$args = wp_parse_args( $args, $defaults );

		$image_list = array();
		$image_booleans = array();
		$image_booleans['gallery'] = 0;

		$from_featured_image = Jetpack_PostImages::from_thumbnail( $post->ID, $args['width'], $args['height'] );
		if ( !empty( $from_featured_image ) ) {
			$srcs = wp_list_pluck( $from_featured_image, 'src' );
			$image_list = array_merge( $image_list, $srcs );
		}

		$from_slideshow = Jetpack_PostImages::from_slideshow( $post->ID, $args['width'], $args['height'] );
		if ( !empty( $from_slideshow ) ) {
			$srcs = wp_list_pluck( $from_slideshow, 'src' );
			$image_list = array_merge( $image_list, $srcs );
		}

		$from_gallery = Jetpack_PostImages::from_gallery( $post->ID );
		if ( !empty( $from_gallery ) ) {
			$srcs = wp_list_pluck( $from_gallery, 'src' );
			$image_list = array_merge( $image_list, $srcs );
			$image_booleans['gallery']++; // @todo This count isn't correct, will only every count 1
		}

		// @todo Can we check width/height of these efficiently?  Could maybe use query args at least, before we strip them out
		$image_list = Jetpack_Media_Meta_Extractor::get_images_from_html( $post->post_content, $image_list );

		return Jetpack_Media_Meta_Extractor::build_image_struct( $image_list, $image_booleans );
	}

	public static function extract_images_from_content( $content, $image_list ) {
		$image_list = Jetpack_Media_Meta_Extractor::get_images_from_html( $content, $image_list );
		return Jetpack_Media_Meta_Extractor::build_image_struct( $image_list, array() );
	}

	public static function build_image_struct( $image_list, $image_booleans ) {
		if ( ! empty( $image_list ) ) {
			$retval = array( 'image' => array() );
			$image_list = array_unique( $image_list );
			foreach ( $image_list as $img ) {
				$retval['image'][] = array( 'url' => $img );
			}
			$image_booleans['image'] = count( $retval['image'] );
			if ( ! empty( $image_booleans ) )
				$retval['has'] = $image_booleans;
			return $retval;
		} else {
			return array();
		}
	}

	/**
	 *
	 * @param string $html Some markup, possibly containing image tags
	 * @param array $images_already_extracted (just an array of image URLs without query strings, no special structure), used for de-duplication
	 * @return array Image URLs extracted from the HTML, stripped of query params and de-duped
	 */
	public static function get_images_from_html( $html, $images_already_extracted ) {
		$image_list = $images_already_extracted;
		$from_html = Jetpack_PostImages::from_html( $html );
		if ( !empty( $from_html ) ) {
			$srcs = wp_list_pluck( $from_html, 'src' );
			foreach( $srcs as $image_url ) {
				if ( ( $src = parse_url( $image_url ) ) && isset( $src['scheme'], $src['host'], $src['path'] ) ) {
					// Rebuild the URL without the query string
					$queryless = $src['scheme'] . '://' . $src['host'] . $src['path'];
				} elseif ( $length = strpos( $image_url, '?' ) ) {
					// If parse_url() didn't work, strip off the query string the old fashioned way
					$queryless = substr( $image_url, 0, $length );
				} else {
					// Failing that, there was no spoon! Err ... query string!
					$queryless = $image_url;
				}

				// Discard URLs that are longer then 4KB, these are likely data URIs or malformed HTML.
				if ( 4096 < strlen( $queryless ) ) {
					continue;
				}

				if ( ! in_array( $queryless, $image_list ) ) {
					$image_list[] = $queryless;
				}
			}
		}
		return $image_list;
	}

	private static function get_stripped_content( $content ) {
		$clean_content = strip_tags( $content );
		$clean_content = html_entity_decode( $clean_content );
		//completely strip shortcodes and any content they enclose
		$clean_content = strip_shortcodes( $clean_content );
		return $clean_content;
	}
}