1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* VMware Balloon driver.
4	*
5	* Copyright (C) 2000-2018, VMware, Inc. All Rights Reserved.
6	*
7	* This is VMware physical memory management driver for Linux. The driver
8	* acts like a "balloon" that can be inflated to reclaim physical pages by
9	* reserving them in the guest and invalidating them in the monitor,
10	* freeing up the underlying machine pages so they can be allocated to
11	* other guests. The balloon can also be deflated to allow the guest to
12	* use more physical memory. Higher level policies can control the sizes
13	* of balloons in VMs in order to manage physical memory resources.
14	*/
15
16	//#define DEBUG
17	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
18
19	#include <linux/types.h>
20	#include <linux/io.h>
21	#include <linux/kernel.h>
22	#include <linux/mm.h>
23	#include <linux/vmalloc.h>
24	#include <linux/sched.h>
25	#include <linux/module.h>
26	#include <linux/workqueue.h>
27	#include <linux/debugfs.h>
28	#include <linux/seq_file.h>
29	#include <linux/rwsem.h>
30	#include <linux/slab.h>
31	#include <linux/spinlock.h>
32	#include <linux/balloon_compaction.h>
33	#include <linux/vmw_vmci_defs.h>
34	#include <linux/vmw_vmci_api.h>
35	#include <asm/hypervisor.h>
36
37	MODULE_AUTHOR("VMware, Inc.");
38	MODULE_DESCRIPTION("VMware Memory Control (Balloon) Driver");
39	MODULE_ALIAS("dmi:*:svnVMware*:*");
40	MODULE_ALIAS("vmware_vmmemctl");
41	MODULE_LICENSE("GPL");
42
43	static bool __read_mostly vmwballoon_shrinker_enable;
44	module_param(vmwballoon_shrinker_enable, bool, 0444);
45	MODULE_PARM_DESC(vmwballoon_shrinker_enable,
46	"Enable non-cooperative out-of-memory protection. Disabled by default as it may degrade performance.");
47
48	/* Delay in seconds after shrink before inflation. */
49	#define VMBALLOON_SHRINK_DELAY (5)
50
51	/* Maximum number of refused pages we accumulate during inflation cycle */
52	#define VMW_BALLOON_MAX_REFUSED 16
53
54	/* Magic number for the balloon mount-point */
55	#define BALLOON_VMW_MAGIC 0x0ba11007
56
57	/*
58	* Hypervisor communication port definitions.
59	*/
60	#define VMW_BALLOON_HV_PORT 0x5670
61	#define VMW_BALLOON_HV_MAGIC 0x456c6d6f
62	#define VMW_BALLOON_GUEST_ID 1 /* Linux */
63
64	enum vmwballoon_capabilities {
65	/*
66	* Bit 0 is reserved and not associated to any capability.
67	*/
68	VMW_BALLOON_BASIC_CMDS = (1 << 1),
69	VMW_BALLOON_BATCHED_CMDS = (1 << 2),
70	VMW_BALLOON_BATCHED_2M_CMDS = (1 << 3),
71	VMW_BALLOON_SIGNALLED_WAKEUP_CMD = (1 << 4),
72	VMW_BALLOON_64_BIT_TARGET = (1 << 5)
73	};
74
75	#define VMW_BALLOON_CAPABILITIES_COMMON (VMW_BALLOON_BASIC_CMDS \
76	| VMW_BALLOON_BATCHED_CMDS \
77	| VMW_BALLOON_BATCHED_2M_CMDS \
78	| VMW_BALLOON_SIGNALLED_WAKEUP_CMD)
79
80	#define VMW_BALLOON_2M_ORDER (PMD_SHIFT - PAGE_SHIFT)
81
82	/*
83	* 64-bit targets are only supported in 64-bit
84	*/
85	#ifdef CONFIG_64BIT
86	#define VMW_BALLOON_CAPABILITIES (VMW_BALLOON_CAPABILITIES_COMMON \
87	| VMW_BALLOON_64_BIT_TARGET)
88	#else
89	#define VMW_BALLOON_CAPABILITIES VMW_BALLOON_CAPABILITIES_COMMON
90	#endif
91
92	enum vmballoon_page_size_type {
93	VMW_BALLOON_4K_PAGE,
94	VMW_BALLOON_2M_PAGE,
95	VMW_BALLOON_LAST_SIZE = VMW_BALLOON_2M_PAGE
96	};
97
98	#define VMW_BALLOON_NUM_PAGE_SIZES (VMW_BALLOON_LAST_SIZE + 1)
99
100	static const char * const vmballoon_page_size_names[] = {
101	[VMW_BALLOON_4K_PAGE] = "4k",
102	[VMW_BALLOON_2M_PAGE] = "2M"
103	};
104
105	enum vmballoon_op {
106	VMW_BALLOON_INFLATE,
107	VMW_BALLOON_DEFLATE
108	};
109
110	enum vmballoon_op_stat_type {
111	VMW_BALLOON_OP_STAT,
112	VMW_BALLOON_OP_FAIL_STAT
113	};
114
115	#define VMW_BALLOON_OP_STAT_TYPES (VMW_BALLOON_OP_FAIL_STAT + 1)
116
117	/**
118	* enum vmballoon_cmd_type - backdoor commands.
119	*
120	* Availability of the commands is as followed:
121	*
122	* %VMW_BALLOON_CMD_START, %VMW_BALLOON_CMD_GET_TARGET and
123	* %VMW_BALLOON_CMD_GUEST_ID are always available.
124	*
125	* If the host reports %VMW_BALLOON_BASIC_CMDS are supported then
126	* %VMW_BALLOON_CMD_LOCK and %VMW_BALLOON_CMD_UNLOCK commands are available.
127	*
128	* If the host reports %VMW_BALLOON_BATCHED_CMDS are supported then
129	* %VMW_BALLOON_CMD_BATCHED_LOCK and VMW_BALLOON_CMD_BATCHED_UNLOCK commands
130	* are available.
131	*
132	* If the host reports %VMW_BALLOON_BATCHED_2M_CMDS are supported then
133	* %VMW_BALLOON_CMD_BATCHED_2M_LOCK and %VMW_BALLOON_CMD_BATCHED_2M_UNLOCK
134	* are supported.
135	*
136	* If the host reports VMW_BALLOON_SIGNALLED_WAKEUP_CMD is supported then
137	* VMW_BALLOON_CMD_VMCI_DOORBELL_SET command is supported.
138	*
139	* @VMW_BALLOON_CMD_START: Communicating supported version with the hypervisor.
140	* @VMW_BALLOON_CMD_GET_TARGET: Gets the balloon target size.
141	* @VMW_BALLOON_CMD_LOCK: Informs the hypervisor about a ballooned page.
142	* @VMW_BALLOON_CMD_UNLOCK: Informs the hypervisor about a page that is about
143	* to be deflated from the balloon.
144	* @VMW_BALLOON_CMD_GUEST_ID: Informs the hypervisor about the type of OS that
145	* runs in the VM.
146	* @VMW_BALLOON_CMD_BATCHED_LOCK: Inform the hypervisor about a batch of
147	* ballooned pages (up to 512).
148	* @VMW_BALLOON_CMD_BATCHED_UNLOCK: Inform the hypervisor about a batch of
149	* pages that are about to be deflated from the
150	* balloon (up to 512).
151	* @VMW_BALLOON_CMD_BATCHED_2M_LOCK: Similar to @VMW_BALLOON_CMD_BATCHED_LOCK
152	* for 2MB pages.
153	* @VMW_BALLOON_CMD_BATCHED_2M_UNLOCK: Similar to
154	* @VMW_BALLOON_CMD_BATCHED_UNLOCK for 2MB
155	* pages.
156	* @VMW_BALLOON_CMD_VMCI_DOORBELL_SET: A command to set doorbell notification
157	* that would be invoked when the balloon
158	* size changes.
159	* @VMW_BALLOON_CMD_LAST: Value of the last command.
160	*/
161	enum vmballoon_cmd_type {
162	VMW_BALLOON_CMD_START,
163	VMW_BALLOON_CMD_GET_TARGET,
164	VMW_BALLOON_CMD_LOCK,
165	VMW_BALLOON_CMD_UNLOCK,
166	VMW_BALLOON_CMD_GUEST_ID,
167	/* No command 5 */
168	VMW_BALLOON_CMD_BATCHED_LOCK = 6,
169	VMW_BALLOON_CMD_BATCHED_UNLOCK,
170	VMW_BALLOON_CMD_BATCHED_2M_LOCK,
171	VMW_BALLOON_CMD_BATCHED_2M_UNLOCK,
172	VMW_BALLOON_CMD_VMCI_DOORBELL_SET,
173	VMW_BALLOON_CMD_LAST = VMW_BALLOON_CMD_VMCI_DOORBELL_SET,
174	};
175
176	#define VMW_BALLOON_CMD_NUM (VMW_BALLOON_CMD_LAST + 1)
177
178	enum vmballoon_error_codes {
179	VMW_BALLOON_SUCCESS,
180	VMW_BALLOON_ERROR_CMD_INVALID,
181	VMW_BALLOON_ERROR_PPN_INVALID,
182	VMW_BALLOON_ERROR_PPN_LOCKED,
183	VMW_BALLOON_ERROR_PPN_UNLOCKED,
184	VMW_BALLOON_ERROR_PPN_PINNED,
185	VMW_BALLOON_ERROR_PPN_NOTNEEDED,
186	VMW_BALLOON_ERROR_RESET,
187	VMW_BALLOON_ERROR_BUSY
188	};
189
190	#define VMW_BALLOON_SUCCESS_WITH_CAPABILITIES (0x03000000)
191
192	#define VMW_BALLOON_CMD_WITH_TARGET_MASK \
193	((1UL << VMW_BALLOON_CMD_GET_TARGET) | \
194	(1UL << VMW_BALLOON_CMD_LOCK) | \
195	(1UL << VMW_BALLOON_CMD_UNLOCK) | \
196	(1UL << VMW_BALLOON_CMD_BATCHED_LOCK) | \
197	(1UL << VMW_BALLOON_CMD_BATCHED_UNLOCK) | \
198	(1UL << VMW_BALLOON_CMD_BATCHED_2M_LOCK) | \
199	(1UL << VMW_BALLOON_CMD_BATCHED_2M_UNLOCK))
200
201	static const char * const vmballoon_cmd_names[] = {
202	[VMW_BALLOON_CMD_START] = "start",
203	[VMW_BALLOON_CMD_GET_TARGET] = "target",
204	[VMW_BALLOON_CMD_LOCK] = "lock",
205	[VMW_BALLOON_CMD_UNLOCK] = "unlock",
206	[VMW_BALLOON_CMD_GUEST_ID] = "guestType",
207	[VMW_BALLOON_CMD_BATCHED_LOCK] = "batchLock",
208	[VMW_BALLOON_CMD_BATCHED_UNLOCK] = "batchUnlock",
209	[VMW_BALLOON_CMD_BATCHED_2M_LOCK] = "2m-lock",
210	[VMW_BALLOON_CMD_BATCHED_2M_UNLOCK] = "2m-unlock",
211	[VMW_BALLOON_CMD_VMCI_DOORBELL_SET] = "doorbellSet"
212	};
213
214	enum vmballoon_stat_page {
215	VMW_BALLOON_PAGE_STAT_ALLOC,
216	VMW_BALLOON_PAGE_STAT_ALLOC_FAIL,
217	VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC,
218	VMW_BALLOON_PAGE_STAT_REFUSED_FREE,
219	VMW_BALLOON_PAGE_STAT_FREE,
220	VMW_BALLOON_PAGE_STAT_LAST = VMW_BALLOON_PAGE_STAT_FREE
221	};
222
223	#define VMW_BALLOON_PAGE_STAT_NUM (VMW_BALLOON_PAGE_STAT_LAST + 1)
224
225	enum vmballoon_stat_general {
226	VMW_BALLOON_STAT_TIMER,
227	VMW_BALLOON_STAT_DOORBELL,
228	VMW_BALLOON_STAT_RESET,
229	VMW_BALLOON_STAT_SHRINK,
230	VMW_BALLOON_STAT_SHRINK_FREE,
231	VMW_BALLOON_STAT_LAST = VMW_BALLOON_STAT_SHRINK_FREE
232	};
233
234	#define VMW_BALLOON_STAT_NUM (VMW_BALLOON_STAT_LAST + 1)
235
236	static DEFINE_STATIC_KEY_TRUE(vmw_balloon_batching);
237	static DEFINE_STATIC_KEY_FALSE(balloon_stat_enabled);
238
239	struct vmballoon_ctl {
240	struct list_head pages;
241	struct list_head refused_pages;
242	struct list_head prealloc_pages;
243	unsigned int n_refused_pages;
244	unsigned int n_pages;
245	enum vmballoon_page_size_type page_size;
246	enum vmballoon_op op;
247	};
248
249	/**
250	* struct vmballoon_batch_entry - a batch entry for lock or unlock.
251	*
252	* @status: the status of the operation, which is written by the hypervisor.
253	* @reserved: reserved for future use. Must be set to zero.
254	* @pfn: the physical frame number of the page to be locked or unlocked.
255	*/
256	struct vmballoon_batch_entry {
257	u64 status : 5;
258	u64 reserved : PAGE_SHIFT - 5;
259	u64 pfn : 52;
260	} __packed;
261
262	struct vmballoon {
263	/**
264	* @max_page_size: maximum supported page size for ballooning.
265	*
266	* Protected by @conf_sem
267	*/
268	enum vmballoon_page_size_type max_page_size;
269
270	/**
271	* @size: balloon actual size in basic page size (frames).
272	*
273	* While we currently do not support size which is bigger than 32-bit,
274	* in preparation for future support, use 64-bits.
275	*/
276	atomic64_t size;
277
278	/**
279	* @target: balloon target size in basic page size (frames).
280	*
281	* We do not protect the target under the assumption that setting the
282	* value is always done through a single write. If this assumption ever
283	* breaks, we would have to use X_ONCE for accesses, and suffer the less
284	* optimized code. Although we may read stale target value if multiple
285	* accesses happen at once, the performance impact should be minor.
286	*/
287	unsigned long target;
288
289	/**
290	* @reset_required: reset flag
291	*
292	* Setting this flag may introduce races, but the code is expected to
293	* handle them gracefully. In the worst case, another operation will
294	* fail as reset did not take place. Clearing the flag is done while
295	* holding @conf_sem for write.
296	*/
297	bool reset_required;
298
299	/**
300	* @capabilities: hypervisor balloon capabilities.
301	*
302	* Protected by @conf_sem.
303	*/
304	unsigned long capabilities;
305
306	/**
307	* @batch_page: pointer to communication batch page.
308	*
309	* When batching is used, batch_page points to a page, which holds up to
310	* %VMW_BALLOON_BATCH_MAX_PAGES entries for locking or unlocking.
311	*/
312	struct vmballoon_batch_entry *batch_page;
313
314	/**
315	* @batch_max_pages: maximum pages that can be locked/unlocked.
316	*
317	* Indicates the number of pages that the hypervisor can lock or unlock
318	* at once, according to whether batching is enabled. If batching is
319	* disabled, only a single page can be locked/unlock on each operation.
320	*
321	* Protected by @conf_sem.
322	*/
323	unsigned int batch_max_pages;
324
325	/**
326	* @page: page to be locked/unlocked by the hypervisor
327	*
328	* @page is only used when batching is disabled and a single page is
329	* reclaimed on each iteration.
330	*
331	* Protected by @comm_lock.
332	*/
333	struct page *page;
334
335	/**
336	* @shrink_timeout: timeout until the next inflation.
337	*
338	* After an shrink event, indicates the time in jiffies after which
339	* inflation is allowed again. Can be written concurrently with reads,
340	* so must use READ_ONCE/WRITE_ONCE when accessing.
341	*/
342	unsigned long shrink_timeout;
343
344	/* statistics */
345	struct vmballoon_stats *stats;
346
347	/**
348	* @b_dev_info: balloon device information descriptor.
349	*/
350	struct balloon_dev_info b_dev_info;
351
352	struct delayed_work dwork;
353
354	/**
355	* @huge_pages - list of the inflated 2MB pages.
356	*
357	* Protected by @b_dev_info.pages_lock .
358	*/
359	struct list_head huge_pages;
360
361	/**
362	* @vmci_doorbell.
363	*
364	* Protected by @conf_sem.
365	*/
366	struct vmci_handle vmci_doorbell;
367
368	/**
369	* @conf_sem: semaphore to protect the configuration and the statistics.
370	*/
371	struct rw_semaphore conf_sem;
372
373	/**
374	* @comm_lock: lock to protect the communication with the host.
375	*
376	* Lock ordering: @conf_sem -> @comm_lock .
377	*/
378	spinlock_t comm_lock;
379
380	/**
381	* @shrinker: shrinker interface that is used to avoid over-inflation.
382	*/
383	struct shrinker *shrinker;
384	};
385
386	static struct vmballoon balloon;
387
388	struct vmballoon_stats {
389	/* timer / doorbell operations */
390	atomic64_t general_stat[VMW_BALLOON_STAT_NUM];
391
392	/* allocation statistics for huge and small pages */
393	atomic64_t
394	page_stat[VMW_BALLOON_PAGE_STAT_NUM][VMW_BALLOON_NUM_PAGE_SIZES];
395
396	/* Monitor operations: total operations, and failures */
397	atomic64_t ops[VMW_BALLOON_CMD_NUM][VMW_BALLOON_OP_STAT_TYPES];
398	};
399
400	static inline bool is_vmballoon_stats_on(void)
401	{
402	return IS_ENABLED(CONFIG_DEBUG_FS) &&
403	static_branch_unlikely(&balloon_stat_enabled);
404	}
405
406	static inline void vmballoon_stats_op_inc(struct vmballoon *b, unsigned int op,
407	enum vmballoon_op_stat_type type)
408	{
409	if (is_vmballoon_stats_on())
410	atomic64_inc(v: &b->stats->ops[op][type]);
411	}
412
413	static inline void vmballoon_stats_gen_inc(struct vmballoon *b,
414	enum vmballoon_stat_general stat)
415	{
416	if (is_vmballoon_stats_on())
417	atomic64_inc(v: &b->stats->general_stat[stat]);
418	}
419
420	static inline void vmballoon_stats_gen_add(struct vmballoon *b,
421	enum vmballoon_stat_general stat,
422	unsigned int val)
423	{
424	if (is_vmballoon_stats_on())
425	atomic64_add(i: val, v: &b->stats->general_stat[stat]);
426	}
427
428	static inline void vmballoon_stats_page_inc(struct vmballoon *b,
429	enum vmballoon_stat_page stat,
430	enum vmballoon_page_size_type size)
431	{
432	if (is_vmballoon_stats_on())
433	atomic64_inc(v: &b->stats->page_stat[stat][size]);
434	}
435
436	static inline void vmballoon_stats_page_add(struct vmballoon *b,
437	enum vmballoon_stat_page stat,
438	enum vmballoon_page_size_type size,
439	unsigned int val)
440	{
441	if (is_vmballoon_stats_on())
442	atomic64_add(i: val, v: &b->stats->page_stat[stat][size]);
443	}
444
445	static inline unsigned long
446	__vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1,
447	unsigned long arg2, unsigned long *result)
448	{
449	unsigned long status, dummy1, dummy2, dummy3, local_result;
450
451	vmballoon_stats_op_inc(b, op: cmd, type: VMW_BALLOON_OP_STAT);
452
453	asm volatile ("inl %%dx" :
454	"=a"(status),
455	"=c"(dummy1),
456	"=d"(dummy2),
457	"=b"(local_result),
458	"=S"(dummy3) :
459	"0"(VMW_BALLOON_HV_MAGIC),
460	"1"(cmd),
461	"2"(VMW_BALLOON_HV_PORT),
462	"3"(arg1),
463	"4"(arg2) :
464	"memory");
465
466	/* update the result if needed */
467	if (result)
468	*result = (cmd == VMW_BALLOON_CMD_START) ? dummy1 :
469	local_result;
470
471	/* update target when applicable */
472	if (status == VMW_BALLOON_SUCCESS &&
473	((1ul << cmd) & VMW_BALLOON_CMD_WITH_TARGET_MASK))
474	WRITE_ONCE(b->target, local_result);
475
476	if (status != VMW_BALLOON_SUCCESS &&
477	status != VMW_BALLOON_SUCCESS_WITH_CAPABILITIES) {
478	vmballoon_stats_op_inc(b, op: cmd, type: VMW_BALLOON_OP_FAIL_STAT);
479	pr_debug("%s: %s [0x%lx,0x%lx) failed, returned %ld\n",
480	__func__, vmballoon_cmd_names[cmd], arg1, arg2,
481	status);
482	}
483
484	/* mark reset required accordingly */
485	if (status == VMW_BALLOON_ERROR_RESET)
486	b->reset_required = true;
487
488	return status;
489	}
490
491	static __always_inline unsigned long
492	vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1,
493	unsigned long arg2)
494	{
495	unsigned long dummy;
496
497	return __vmballoon_cmd(b, cmd, arg1, arg2, result: &dummy);
498	}
499
500	/*
501	* Send "start" command to the host, communicating supported version
502	* of the protocol.
503	*/
504	static int vmballoon_send_start(struct vmballoon *b, unsigned long req_caps)
505	{
506	unsigned long status, capabilities;
507
508	status = __vmballoon_cmd(b, cmd: VMW_BALLOON_CMD_START, arg1: req_caps, arg2: 0,
509	result: &capabilities);
510
511	switch (status) {
512	case VMW_BALLOON_SUCCESS_WITH_CAPABILITIES:
513	b->capabilities = capabilities;
514	break;
515	case VMW_BALLOON_SUCCESS:
516	b->capabilities = VMW_BALLOON_BASIC_CMDS;
517	break;
518	default:
519	return -EIO;
520	}
521
522	/*
523	* 2MB pages are only supported with batching. If batching is for some
524	* reason disabled, do not use 2MB pages, since otherwise the legacy
525	* mechanism is used with 2MB pages, causing a failure.
526	*/
527	b->max_page_size = VMW_BALLOON_4K_PAGE;
528	if ((b->capabilities & VMW_BALLOON_BATCHED_2M_CMDS) &&
529	(b->capabilities & VMW_BALLOON_BATCHED_CMDS))
530	b->max_page_size = VMW_BALLOON_2M_PAGE;
531
532
533	return 0;
534	}
535
536	/**
537	* vmballoon_send_guest_id - communicate guest type to the host.
538	*
539	* @b: pointer to the balloon.
540	*
541	* Communicate guest type to the host so that it can adjust ballooning
542	* algorithm to the one most appropriate for the guest. This command
543	* is normally issued after sending "start" command and is part of
544	* standard reset sequence.
545	*
546	* Return: zero on success or appropriate error code.
547	*/
548	static int vmballoon_send_guest_id(struct vmballoon *b)
549	{
550	unsigned long status;
551
552	status = vmballoon_cmd(b, cmd: VMW_BALLOON_CMD_GUEST_ID,
553	VMW_BALLOON_GUEST_ID, arg2: 0);
554
555	return status == VMW_BALLOON_SUCCESS ? 0 : -EIO;
556	}
557
558	/**
559	* vmballoon_page_order() - return the order of the page
560	* @page_size: the size of the page.
561	*
562	* Return: the allocation order.
563	*/
564	static inline
565	unsigned int vmballoon_page_order(enum vmballoon_page_size_type page_size)
566	{
567	return page_size == VMW_BALLOON_2M_PAGE ? VMW_BALLOON_2M_ORDER : 0;
568	}
569
570	/**
571	* vmballoon_page_in_frames() - returns the number of frames in a page.
572	* @page_size: the size of the page.
573	*
574	* Return: the number of 4k frames.
575	*/
576	static inline unsigned int
577	vmballoon_page_in_frames(enum vmballoon_page_size_type page_size)
578	{
579	return 1 << vmballoon_page_order(page_size);
580	}
581
582	/**
583	* vmballoon_mark_page_offline() - mark a page as offline
584	* @page: pointer for the page.
585	* @page_size: the size of the page.
586	*/
587	static void
588	vmballoon_mark_page_offline(struct page *page,
589	enum vmballoon_page_size_type page_size)
590	{
591	int i;
592
593	for (i = 0; i < vmballoon_page_in_frames(page_size); i++)
594	__SetPageOffline(page: page + i);
595	}
596
597	/**
598	* vmballoon_mark_page_online() - mark a page as online
599	* @page: pointer for the page.
600	* @page_size: the size of the page.
601	*/
602	static void
603	vmballoon_mark_page_online(struct page *page,
604	enum vmballoon_page_size_type page_size)
605	{
606	int i;
607
608	for (i = 0; i < vmballoon_page_in_frames(page_size); i++)
609	__ClearPageOffline(page: page + i);
610	}
611
612	/**
613	* vmballoon_send_get_target() - Retrieve desired balloon size from the host.
614	*
615	* @b: pointer to the balloon.
616	*
617	* Return: zero on success, EINVAL if limit does not fit in 32-bit, as required
618	* by the host-guest protocol and EIO if an error occurred in communicating with
619	* the host.
620	*/
621	static int vmballoon_send_get_target(struct vmballoon *b)
622	{
623	unsigned long status;
624	unsigned long limit;
625
626	limit = totalram_pages();
627
628	/* Ensure limit fits in 32-bits if 64-bit targets are not supported */
629	if (!(b->capabilities & VMW_BALLOON_64_BIT_TARGET) &&
630	limit != (u32)limit)
631	return -EINVAL;
632
633	status = vmballoon_cmd(b, cmd: VMW_BALLOON_CMD_GET_TARGET, arg1: limit, arg2: 0);
634
635	return status == VMW_BALLOON_SUCCESS ? 0 : -EIO;
636	}
637
638	/**
639	* vmballoon_alloc_page_list - allocates a list of pages.
640	*
641	* @b: pointer to the balloon.
642	* @ctl: pointer for the %struct vmballoon_ctl, which defines the operation.
643	* @req_n_pages: the number of requested pages.
644	*
645	* Tries to allocate @req_n_pages. Add them to the list of balloon pages in
646	* @ctl.pages and updates @ctl.n_pages to reflect the number of pages.
647	*
648	* Return: zero on success or error code otherwise.
649	*/
650	static int vmballoon_alloc_page_list(struct vmballoon *b,
651	struct vmballoon_ctl *ctl,
652	unsigned int req_n_pages)
653	{
654	struct page *page;
655	unsigned int i;
656
657	for (i = 0; i < req_n_pages; i++) {
658	/*
659	* First check if we happen to have pages that were allocated
660	* before. This happens when 2MB page rejected during inflation
661	* by the hypervisor, and then split into 4KB pages.
662	*/
663	if (!list_empty(head: &ctl->prealloc_pages)) {
664	page = list_first_entry(&ctl->prealloc_pages,
665	struct page, lru);
666	list_del(entry: &page->lru);
667	} else {
668	if (ctl->page_size == VMW_BALLOON_2M_PAGE)
669	page = alloc_pages(__GFP_HIGHMEM|__GFP_NOWARN|
670	__GFP_NOMEMALLOC, VMW_BALLOON_2M_ORDER);
671	else
672	page = balloon_page_alloc();
673
674	vmballoon_stats_page_inc(b, stat: VMW_BALLOON_PAGE_STAT_ALLOC,
675	size: ctl->page_size);
676	}
677
678	if (page) {
679	/* Success. Add the page to the list and continue. */
680	list_add(new: &page->lru, head: &ctl->pages);
681	continue;
682	}
683
684	/* Allocation failed. Update statistics and stop. */
685	vmballoon_stats_page_inc(b, stat: VMW_BALLOON_PAGE_STAT_ALLOC_FAIL,
686	size: ctl->page_size);
687	break;
688	}
689
690	ctl->n_pages = i;
691
692	return req_n_pages == ctl->n_pages ? 0 : -ENOMEM;
693	}
694
695	/**
696	* vmballoon_handle_one_result - Handle lock/unlock result for a single page.
697	*
698	* @b: pointer for %struct vmballoon.
699	* @page: pointer for the page whose result should be handled.
700	* @page_size: size of the page.
701	* @status: status of the operation as provided by the hypervisor.
702	*/
703	static int vmballoon_handle_one_result(struct vmballoon *b, struct page *page,
704	enum vmballoon_page_size_type page_size,
705	unsigned long status)
706	{
707	/* On success do nothing. The page is already on the balloon list. */
708	if (likely(status == VMW_BALLOON_SUCCESS))
709	return 0;
710
711	pr_debug("%s: failed comm pfn %lx status %lu page_size %s\n", __func__,
712	page_to_pfn(page), status,
713	vmballoon_page_size_names[page_size]);
714
715	/* Error occurred */
716	vmballoon_stats_page_inc(b, stat: VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC,
717	size: page_size);
718
719	return -EIO;
720	}
721
722	/**
723	* vmballoon_status_page - returns the status of (un)lock operation
724	*
725	* @b: pointer to the balloon.
726	* @idx: index for the page for which the operation is performed.
727	* @p: pointer to where the page struct is returned.
728	*
729	* Following a lock or unlock operation, returns the status of the operation for
730	* an individual page. Provides the page that the operation was performed on on
731	* the @page argument.
732	*
733	* Returns: The status of a lock or unlock operation for an individual page.
734	*/
735	static unsigned long vmballoon_status_page(struct vmballoon *b, int idx,
736	struct page **p)
737	{
738	if (static_branch_likely(&vmw_balloon_batching)) {
739	/* batching mode */
740	*p = pfn_to_page(b->batch_page[idx].pfn);
741	return b->batch_page[idx].status;
742	}
743
744	/* non-batching mode */
745	*p = b->page;
746
747	/*
748	* If a failure occurs, the indication will be provided in the status
749	* of the entire operation, which is considered before the individual
750	* page status. So for non-batching mode, the indication is always of
751	* success.
752	*/
753	return VMW_BALLOON_SUCCESS;
754	}
755
756	/**
757	* vmballoon_lock_op - notifies the host about inflated/deflated pages.
758	* @b: pointer to the balloon.
759	* @num_pages: number of inflated/deflated pages.
760	* @page_size: size of the page.
761	* @op: the type of operation (lock or unlock).
762	*
763	* Notify the host about page(s) that were ballooned (or removed from the
764	* balloon) so that host can use it without fear that guest will need it (or
765	* stop using them since the VM does). Host may reject some pages, we need to
766	* check the return value and maybe submit a different page. The pages that are
767	* inflated/deflated are pointed by @b->page.
768	*
769	* Return: result as provided by the hypervisor.
770	*/
771	static unsigned long vmballoon_lock_op(struct vmballoon *b,
772	unsigned int num_pages,
773	enum vmballoon_page_size_type page_size,
774	enum vmballoon_op op)
775	{
776	unsigned long cmd, pfn;
777
778	lockdep_assert_held(&b->comm_lock);
779
780	if (static_branch_likely(&vmw_balloon_batching)) {
781	if (op == VMW_BALLOON_INFLATE)
782	cmd = page_size == VMW_BALLOON_2M_PAGE ?
783	VMW_BALLOON_CMD_BATCHED_2M_LOCK :
784	VMW_BALLOON_CMD_BATCHED_LOCK;
785	else
786	cmd = page_size == VMW_BALLOON_2M_PAGE ?
787	VMW_BALLOON_CMD_BATCHED_2M_UNLOCK :
788	VMW_BALLOON_CMD_BATCHED_UNLOCK;
789
790	pfn = PHYS_PFN(virt_to_phys(b->batch_page));
791	} else {
792	cmd = op == VMW_BALLOON_INFLATE ? VMW_BALLOON_CMD_LOCK :
793	VMW_BALLOON_CMD_UNLOCK;
794	pfn = page_to_pfn(b->page);
795
796	/* In non-batching mode, PFNs must fit in 32-bit */
797	if (unlikely(pfn != (u32)pfn))
798	return VMW_BALLOON_ERROR_PPN_INVALID;
799	}
800
801	return vmballoon_cmd(b, cmd, arg1: pfn, arg2: num_pages);
802	}
803
804	/**
805	* vmballoon_add_page - adds a page towards lock/unlock operation.
806	*
807	* @b: pointer to the balloon.
808	* @idx: index of the page to be ballooned in this batch.
809	* @p: pointer to the page that is about to be ballooned.
810	*
811	* Adds the page to be ballooned. Must be called while holding @comm_lock.
812	*/
813	static void vmballoon_add_page(struct vmballoon *b, unsigned int idx,
814	struct page *p)
815	{
816	lockdep_assert_held(&b->comm_lock);
817
818	if (static_branch_likely(&vmw_balloon_batching))
819	b->batch_page[idx] = (struct vmballoon_batch_entry)
820	{ .pfn = page_to_pfn(p) };
821	else
822	b->page = p;
823	}
824
825	/**
826	* vmballoon_lock - lock or unlock a batch of pages.
827	*
828	* @b: pointer to the balloon.
829	* @ctl: pointer for the %struct vmballoon_ctl, which defines the operation.
830	*
831	* Notifies the host of about ballooned pages (after inflation or deflation,
832	* according to @ctl). If the host rejects the page put it on the
833	* @ctl refuse list. These refused page are then released when moving to the
834	* next size of pages.
835	*
836	* Note that we neither free any @page here nor put them back on the ballooned
837	* pages list. Instead we queue it for later processing. We do that for several
838	* reasons. First, we do not want to free the page under the lock. Second, it
839	* allows us to unify the handling of lock and unlock. In the inflate case, the
840	* caller will check if there are too many refused pages and release them.
841	* Although it is not identical to the past behavior, it should not affect
842	* performance.
843	*/
844	static int vmballoon_lock(struct vmballoon *b, struct vmballoon_ctl *ctl)
845	{
846	unsigned long batch_status;
847	struct page *page;
848	unsigned int i, num_pages;
849
850	num_pages = ctl->n_pages;
851	if (num_pages == 0)
852	return 0;
853
854	/* communication with the host is done under the communication lock */
855	spin_lock(lock: &b->comm_lock);
856
857	i = 0;
858	list_for_each_entry(page, &ctl->pages, lru)
859	vmballoon_add_page(b, idx: i++, p: page);
860
861	batch_status = vmballoon_lock_op(b, num_pages: ctl->n_pages, page_size: ctl->page_size,
862	op: ctl->op);
863
864	/*
865	* Iterate over the pages in the provided list. Since we are changing
866	* @ctl->n_pages we are saving the original value in @num_pages and
867	* use this value to bound the loop.
868	*/
869	for (i = 0; i < num_pages; i++) {
870	unsigned long status;
871
872	status = vmballoon_status_page(b, idx: i, p: &page);
873
874	/*
875	* Failure of the whole batch overrides a single operation
876	* results.
877	*/
878	if (batch_status != VMW_BALLOON_SUCCESS)
879	status = batch_status;
880
881	/* Continue if no error happened */
882	if (!vmballoon_handle_one_result(b, page, page_size: ctl->page_size,
883	status))
884	continue;
885
886	/*
887	* Error happened. Move the pages to the refused list and update
888	* the pages number.
889	*/
890	list_move(list: &page->lru, head: &ctl->refused_pages);
891	ctl->n_pages--;
892	ctl->n_refused_pages++;
893	}
894
895	spin_unlock(lock: &b->comm_lock);
896
897	return batch_status == VMW_BALLOON_SUCCESS ? 0 : -EIO;
898	}
899
900	/**
901	* vmballoon_release_page_list() - Releases a page list
902	*
903	* @page_list: list of pages to release.
904	* @n_pages: pointer to the number of pages.
905	* @page_size: whether the pages in the list are 2MB (or else 4KB).
906	*
907	* Releases the list of pages and zeros the number of pages.
908	*/
909	static void vmballoon_release_page_list(struct list_head *page_list,
910	int *n_pages,
911	enum vmballoon_page_size_type page_size)
912	{
913	struct page *page, *tmp;
914
915	list_for_each_entry_safe(page, tmp, page_list, lru) {
916	list_del(entry: &page->lru);
917	__free_pages(page, order: vmballoon_page_order(page_size));
918	}
919
920	if (n_pages)
921	*n_pages = 0;
922	}
923
924
925	/*
926	* Release pages that were allocated while attempting to inflate the
927	* balloon but were refused by the host for one reason or another.
928	*/
929	static void vmballoon_release_refused_pages(struct vmballoon *b,
930	struct vmballoon_ctl *ctl)
931	{
932	vmballoon_stats_page_inc(b, stat: VMW_BALLOON_PAGE_STAT_REFUSED_FREE,
933	size: ctl->page_size);
934
935	vmballoon_release_page_list(page_list: &ctl->refused_pages, n_pages: &ctl->n_refused_pages,
936	page_size: ctl->page_size);
937	}
938
939	/**
940	* vmballoon_change - retrieve the required balloon change
941	*
942	* @b: pointer for the balloon.
943	*
944	* Return: the required change for the balloon size. A positive number
945	* indicates inflation, a negative number indicates a deflation.
946	*/
947	static int64_t vmballoon_change(struct vmballoon *b)
948	{
949	int64_t size, target;
950
951	size = atomic64_read(v: &b->size);
952	target = READ_ONCE(b->target);
953
954	/*
955	* We must cast first because of int sizes
956	* Otherwise we might get huge positives instead of negatives
957	*/
958
959	if (b->reset_required)
960	return 0;
961
962	/* consider a 2MB slack on deflate, unless the balloon is emptied */
963	if (target < size && target != 0 &&
964	size - target < vmballoon_page_in_frames(page_size: VMW_BALLOON_2M_PAGE))
965	return 0;
966
967	/* If an out-of-memory recently occurred, inflation is disallowed. */
968	if (target > size && time_before(jiffies, READ_ONCE(b->shrink_timeout)))
969	return 0;
970
971	return target - size;
972	}
973
974	/**
975	* vmballoon_enqueue_page_list() - Enqueues list of pages after inflation.
976	*
977	* @b: pointer to balloon.
978	* @pages: list of pages to enqueue.
979	* @n_pages: pointer to number of pages in list. The value is zeroed.
980	* @page_size: whether the pages are 2MB or 4KB pages.
981	*
982	* Enqueues the provides list of pages in the ballooned page list, clears the
983	* list and zeroes the number of pages that was provided.
984	*/
985	static void vmballoon_enqueue_page_list(struct vmballoon *b,
986	struct list_head *pages,
987	unsigned int *n_pages,
988	enum vmballoon_page_size_type page_size)
989	{
990	unsigned long flags;
991	struct page *page;
992
993	if (page_size == VMW_BALLOON_4K_PAGE) {
994	balloon_page_list_enqueue(b_dev_info: &b->b_dev_info, pages);
995	} else {
996	/*
997	* Keep the huge pages in a local list which is not available
998	* for the balloon compaction mechanism.
999	*/
1000	spin_lock_irqsave(&b->b_dev_info.pages_lock, flags);
1001
1002	list_for_each_entry(page, pages, lru) {
1003	vmballoon_mark_page_offline(page, page_size: VMW_BALLOON_2M_PAGE);
1004	}
1005
1006	list_splice_init(list: pages, head: &b->huge_pages);
1007	__count_vm_events(item: BALLOON_INFLATE, delta: *n_pages *
1008	vmballoon_page_in_frames(page_size: VMW_BALLOON_2M_PAGE));
1009	spin_unlock_irqrestore(lock: &b->b_dev_info.pages_lock, flags);
1010	}
1011
1012	*n_pages = 0;
1013	}
1014
1015	/**
1016	* vmballoon_dequeue_page_list() - Dequeues page lists for deflation.
1017	*
1018	* @b: pointer to balloon.
1019	* @pages: list of pages to enqueue.
1020	* @n_pages: pointer to number of pages in list. The value is zeroed.
1021	* @page_size: whether the pages are 2MB or 4KB pages.
1022	* @n_req_pages: the number of requested pages.
1023	*
1024	* Dequeues the number of requested pages from the balloon for deflation. The
1025	* number of dequeued pages may be lower, if not enough pages in the requested
1026	* size are available.
1027	*/
1028	static void vmballoon_dequeue_page_list(struct vmballoon *b,
1029	struct list_head *pages,
1030	unsigned int *n_pages,
1031	enum vmballoon_page_size_type page_size,
1032	unsigned int n_req_pages)
1033	{
1034	struct page *page, *tmp;
1035	unsigned int i = 0;
1036	unsigned long flags;
1037
1038	/* In the case of 4k pages, use the compaction infrastructure */
1039	if (page_size == VMW_BALLOON_4K_PAGE) {
1040	*n_pages = balloon_page_list_dequeue(b_dev_info: &b->b_dev_info, pages,
1041	n_req_pages);
1042	return;
1043	}
1044
1045	/* 2MB pages */
1046	spin_lock_irqsave(&b->b_dev_info.pages_lock, flags);
1047	list_for_each_entry_safe(page, tmp, &b->huge_pages, lru) {
1048	vmballoon_mark_page_online(page, page_size: VMW_BALLOON_2M_PAGE);
1049
1050	list_move(list: &page->lru, head: pages);
1051	if (++i == n_req_pages)
1052	break;
1053	}
1054
1055	__count_vm_events(item: BALLOON_DEFLATE,
1056	delta: i * vmballoon_page_in_frames(page_size: VMW_BALLOON_2M_PAGE));
1057	spin_unlock_irqrestore(lock: &b->b_dev_info.pages_lock, flags);
1058	*n_pages = i;
1059	}
1060
1061	/**
1062	* vmballoon_split_refused_pages() - Split the 2MB refused pages to 4k.
1063	*
1064	* If inflation of 2MB pages was denied by the hypervisor, it is likely to be
1065	* due to one or few 4KB pages. These 2MB pages may keep being allocated and
1066	* then being refused. To prevent this case, this function splits the refused
1067	* pages into 4KB pages and adds them into @prealloc_pages list.
1068	*
1069	* @ctl: pointer for the %struct vmballoon_ctl, which defines the operation.
1070	*/
1071	static void vmballoon_split_refused_pages(struct vmballoon_ctl *ctl)
1072	{
1073	struct page *page, *tmp;
1074	unsigned int i, order;
1075
1076	order = vmballoon_page_order(page_size: ctl->page_size);
1077
1078	list_for_each_entry_safe(page, tmp, &ctl->refused_pages, lru) {
1079	list_del(entry: &page->lru);
1080	split_page(page, order);
1081	for (i = 0; i < (1 << order); i++)
1082	list_add(new: &page[i].lru, head: &ctl->prealloc_pages);
1083	}
1084	ctl->n_refused_pages = 0;
1085	}
1086
1087	/**
1088	* vmballoon_inflate() - Inflate the balloon towards its target size.
1089	*
1090	* @b: pointer to the balloon.
1091	*/
1092	static void vmballoon_inflate(struct vmballoon *b)
1093	{
1094	int64_t to_inflate_frames;
1095	struct vmballoon_ctl ctl = {
1096	.pages = LIST_HEAD_INIT(ctl.pages),
1097	.refused_pages = LIST_HEAD_INIT(ctl.refused_pages),
1098	.prealloc_pages = LIST_HEAD_INIT(ctl.prealloc_pages),
1099	.page_size = b->max_page_size,
1100	.op = VMW_BALLOON_INFLATE
1101	};
1102
1103	while ((to_inflate_frames = vmballoon_change(b)) > 0) {
1104	unsigned int to_inflate_pages, page_in_frames;
1105	int alloc_error, lock_error = 0;
1106
1107	VM_BUG_ON(!list_empty(&ctl.pages));
1108	VM_BUG_ON(ctl.n_pages != 0);
1109
1110	page_in_frames = vmballoon_page_in_frames(page_size: ctl.page_size);
1111
1112	to_inflate_pages = min_t(unsigned long, b->batch_max_pages,
1113	DIV_ROUND_UP_ULL(to_inflate_frames,
1114	page_in_frames));
1115
1116	/* Start by allocating */
1117	alloc_error = vmballoon_alloc_page_list(b, ctl: &ctl,
1118	req_n_pages: to_inflate_pages);
1119
1120	/* Actually lock the pages by telling the hypervisor */
1121	lock_error = vmballoon_lock(b, ctl: &ctl);
1122
1123	/*
1124	* If an error indicates that something serious went wrong,
1125	* stop the inflation.
1126	*/
1127	if (lock_error)
1128	break;
1129
1130	/* Update the balloon size */
1131	atomic64_add(i: ctl.n_pages * page_in_frames, v: &b->size);
1132
1133	vmballoon_enqueue_page_list(b, pages: &ctl.pages, n_pages: &ctl.n_pages,
1134	page_size: ctl.page_size);
1135
1136	/*
1137	* If allocation failed or the number of refused pages exceeds
1138	* the maximum allowed, move to the next page size.
1139	*/
1140	if (alloc_error ||
1141	ctl.n_refused_pages >= VMW_BALLOON_MAX_REFUSED) {
1142	if (ctl.page_size == VMW_BALLOON_4K_PAGE)
1143	break;
1144
1145	/*
1146	* Split the refused pages to 4k. This will also empty
1147	* the refused pages list.
1148	*/
1149	vmballoon_split_refused_pages(ctl: &ctl);
1150	ctl.page_size--;
1151	}
1152
1153	cond_resched();
1154	}
1155
1156	/*
1157	* Release pages that were allocated while attempting to inflate the
1158	* balloon but were refused by the host for one reason or another,
1159	* and update the statistics.
1160	*/
1161	if (ctl.n_refused_pages != 0)
1162	vmballoon_release_refused_pages(b, ctl: &ctl);
1163
1164	vmballoon_release_page_list(page_list: &ctl.prealloc_pages, NULL, page_size: ctl.page_size);
1165	}
1166
1167	/**
1168	* vmballoon_deflate() - Decrease the size of the balloon.
1169	*
1170	* @b: pointer to the balloon
1171	* @n_frames: the number of frames to deflate. If zero, automatically
1172	* calculated according to the target size.
1173	* @coordinated: whether to coordinate with the host
1174	*
1175	* Decrease the size of the balloon allowing guest to use more memory.
1176	*
1177	* Return: The number of deflated frames (i.e., basic page size units)
1178	*/
1179	static unsigned long vmballoon_deflate(struct vmballoon *b, uint64_t n_frames,
1180	bool coordinated)
1181	{
1182	unsigned long deflated_frames = 0;
1183	unsigned long tried_frames = 0;
1184	struct vmballoon_ctl ctl = {
1185	.pages = LIST_HEAD_INIT(ctl.pages),
1186	.refused_pages = LIST_HEAD_INIT(ctl.refused_pages),
1187	.page_size = VMW_BALLOON_4K_PAGE,
1188	.op = VMW_BALLOON_DEFLATE
1189	};
1190
1191	/* free pages to reach target */
1192	while (true) {
1193	unsigned int to_deflate_pages, n_unlocked_frames;
1194	unsigned int page_in_frames;
1195	int64_t to_deflate_frames;
1196	bool deflated_all;
1197
1198	page_in_frames = vmballoon_page_in_frames(page_size: ctl.page_size);
1199
1200	VM_BUG_ON(!list_empty(&ctl.pages));
1201	VM_BUG_ON(ctl.n_pages);
1202	VM_BUG_ON(!list_empty(&ctl.refused_pages));
1203	VM_BUG_ON(ctl.n_refused_pages);
1204
1205	/*
1206	* If we were requested a specific number of frames, we try to
1207	* deflate this number of frames. Otherwise, deflation is
1208	* performed according to the target and balloon size.
1209	*/
1210	to_deflate_frames = n_frames ? n_frames - tried_frames :
1211	-vmballoon_change(b);
1212
1213	/* break if no work to do */
1214	if (to_deflate_frames <= 0)
1215	break;
1216
1217	/*
1218	* Calculate the number of frames based on current page size,
1219	* but limit the deflated frames to a single chunk
1220	*/
1221	to_deflate_pages = min_t(unsigned long, b->batch_max_pages,
1222	DIV_ROUND_UP_ULL(to_deflate_frames,
1223	page_in_frames));
1224
1225	/* First take the pages from the balloon pages. */
1226	vmballoon_dequeue_page_list(b, pages: &ctl.pages, n_pages: &ctl.n_pages,
1227	page_size: ctl.page_size, n_req_pages: to_deflate_pages);
1228
1229	/*
1230	* Before pages are moving to the refused list, count their
1231	* frames as frames that we tried to deflate.
1232	*/
1233	tried_frames += ctl.n_pages * page_in_frames;
1234
1235	/*
1236	* Unlock the pages by communicating with the hypervisor if the
1237	* communication is coordinated (i.e., not pop). We ignore the
1238	* return code. Instead we check if all the pages we manage to
1239	* unlock all the pages. If we failed, we will move to the next
1240	* page size, and would eventually try again later.
1241	*/
1242	if (coordinated)
1243	vmballoon_lock(b, ctl: &ctl);
1244
1245	/*
1246	* Check if we deflated enough. We will move to the next page
1247	* size if we did not manage to do so. This calculation takes
1248	* place now, as once the pages are released, the number of
1249	* pages is zeroed.
1250	*/
1251	deflated_all = (ctl.n_pages == to_deflate_pages);
1252
1253	/* Update local and global counters */
1254	n_unlocked_frames = ctl.n_pages * page_in_frames;
1255	atomic64_sub(i: n_unlocked_frames, v: &b->size);
1256	deflated_frames += n_unlocked_frames;
1257
1258	vmballoon_stats_page_add(b, stat: VMW_BALLOON_PAGE_STAT_FREE,
1259	size: ctl.page_size, val: ctl.n_pages);
1260
1261	/* free the ballooned pages */
1262	vmballoon_release_page_list(page_list: &ctl.pages, n_pages: &ctl.n_pages,
1263	page_size: ctl.page_size);
1264
1265	/* Return the refused pages to the ballooned list. */
1266	vmballoon_enqueue_page_list(b, pages: &ctl.refused_pages,
1267	n_pages: &ctl.n_refused_pages,
1268	page_size: ctl.page_size);
1269
1270	/* If we failed to unlock all the pages, move to next size. */
1271	if (!deflated_all) {
1272	if (ctl.page_size == b->max_page_size)
1273	break;
1274	ctl.page_size++;
1275	}
1276
1277	cond_resched();
1278	}
1279
1280	return deflated_frames;
1281	}
1282
1283	/**
1284	* vmballoon_deinit_batching - disables batching mode.
1285	*
1286	* @b: pointer to &struct vmballoon.
1287	*
1288	* Disables batching, by deallocating the page for communication with the
1289	* hypervisor and disabling the static key to indicate that batching is off.
1290	*/
1291	static void vmballoon_deinit_batching(struct vmballoon *b)
1292	{
1293	free_page((unsigned long)b->batch_page);
1294	b->batch_page = NULL;
1295	static_branch_disable(&vmw_balloon_batching);
1296	b->batch_max_pages = 1;
1297	}
1298
1299	/**
1300	* vmballoon_init_batching - enable batching mode.
1301	*
1302	* @b: pointer to &struct vmballoon.
1303	*
1304	* Enables batching, by allocating a page for communication with the hypervisor
1305	* and enabling the static_key to use batching.
1306	*
1307	* Return: zero on success or an appropriate error-code.
1308	*/
1309	static int vmballoon_init_batching(struct vmballoon *b)
1310	{
1311	struct page *page;
1312
1313	page = alloc_page(GFP_KERNEL | __GFP_ZERO);
1314	if (!page)
1315	return -ENOMEM;
1316
1317	b->batch_page = page_address(page);
1318	b->batch_max_pages = PAGE_SIZE / sizeof(struct vmballoon_batch_entry);
1319
1320	static_branch_enable(&vmw_balloon_batching);
1321
1322	return 0;
1323	}
1324
1325	/*
1326	* Receive notification and resize balloon
1327	*/
1328	static void vmballoon_doorbell(void *client_data)
1329	{
1330	struct vmballoon *b = client_data;
1331
1332	vmballoon_stats_gen_inc(b, stat: VMW_BALLOON_STAT_DOORBELL);
1333
1334	mod_delayed_work(wq: system_freezable_wq, dwork: &b->dwork, delay: 0);
1335	}
1336
1337	/*
1338	* Clean up vmci doorbell
1339	*/
1340	static void vmballoon_vmci_cleanup(struct vmballoon *b)
1341	{
1342	vmballoon_cmd(b, cmd: VMW_BALLOON_CMD_VMCI_DOORBELL_SET,
1343	VMCI_INVALID_ID, VMCI_INVALID_ID);
1344
1345	if (!vmci_handle_is_invalid(h: b->vmci_doorbell)) {
1346	vmci_doorbell_destroy(handle: b->vmci_doorbell);
1347	b->vmci_doorbell = VMCI_INVALID_HANDLE;
1348	}
1349	}
1350
1351	/**
1352	* vmballoon_vmci_init - Initialize vmci doorbell.
1353	*
1354	* @b: pointer to the balloon.
1355	*
1356	* Return: zero on success or when wakeup command not supported. Error-code
1357	* otherwise.
1358	*
1359	* Initialize vmci doorbell, to get notified as soon as balloon changes.
1360	*/
1361	static int vmballoon_vmci_init(struct vmballoon *b)
1362	{
1363	unsigned long error;
1364
1365	if ((b->capabilities & VMW_BALLOON_SIGNALLED_WAKEUP_CMD) == 0)
1366	return 0;
1367
1368	error = vmci_doorbell_create(handle: &b->vmci_doorbell, VMCI_FLAG_DELAYED_CB,
1369	priv_flags: VMCI_PRIVILEGE_FLAG_RESTRICTED,
1370	notify_cb: vmballoon_doorbell, client_data: b);
1371
1372	if (error != VMCI_SUCCESS)
1373	goto fail;
1374
1375	error = __vmballoon_cmd(b, cmd: VMW_BALLOON_CMD_VMCI_DOORBELL_SET,
1376	arg1: b->vmci_doorbell.context,
1377	arg2: b->vmci_doorbell.resource, NULL);
1378
1379	if (error != VMW_BALLOON_SUCCESS)
1380	goto fail;
1381
1382	return 0;
1383	fail:
1384	vmballoon_vmci_cleanup(b);
1385	return -EIO;
1386	}
1387
1388	/**
1389	* vmballoon_pop - Quickly release all pages allocate for the balloon.
1390	*
1391	* @b: pointer to the balloon.
1392	*
1393	* This function is called when host decides to "reset" balloon for one reason
1394	* or another. Unlike normal "deflate" we do not (shall not) notify host of the
1395	* pages being released.
1396	*/
1397	static void vmballoon_pop(struct vmballoon *b)
1398	{
1399	unsigned long size;
1400
1401	while ((size = atomic64_read(v: &b->size)))
1402	vmballoon_deflate(b, n_frames: size, coordinated: false);
1403	}
1404
1405	/*
1406	* Perform standard reset sequence by popping the balloon (in case it
1407	* is not empty) and then restarting protocol. This operation normally
1408	* happens when host responds with VMW_BALLOON_ERROR_RESET to a command.
1409	*/
1410	static void vmballoon_reset(struct vmballoon *b)
1411	{
1412	int error;
1413
1414	down_write(sem: &b->conf_sem);
1415
1416	vmballoon_vmci_cleanup(b);
1417
1418	/* free all pages, skipping monitor unlock */
1419	vmballoon_pop(b);
1420
1421	if (vmballoon_send_start(b, VMW_BALLOON_CAPABILITIES))
1422	goto unlock;
1423
1424	if ((b->capabilities & VMW_BALLOON_BATCHED_CMDS) != 0) {
1425	if (vmballoon_init_batching(b)) {
1426	/*
1427	* We failed to initialize batching, inform the monitor
1428	* about it by sending a null capability.
1429	*
1430	* The guest will retry in one second.
1431	*/
1432	vmballoon_send_start(b, req_caps: 0);
1433	goto unlock;
1434	}
1435	} else if ((b->capabilities & VMW_BALLOON_BASIC_CMDS) != 0) {
1436	vmballoon_deinit_batching(b);
1437	}
1438
1439	vmballoon_stats_gen_inc(b, stat: VMW_BALLOON_STAT_RESET);
1440	b->reset_required = false;
1441
1442	error = vmballoon_vmci_init(b);
1443	if (error)
1444	pr_err_once("failed to initialize vmci doorbell\n");
1445
1446	if (vmballoon_send_guest_id(b))
1447	pr_err_once("failed to send guest ID to the host\n");
1448
1449	unlock:
1450	up_write(sem: &b->conf_sem);
1451	}
1452
1453	/**
1454	* vmballoon_work - periodic balloon worker for reset, inflation and deflation.
1455	*
1456	* @work: pointer to the &work_struct which is provided by the workqueue.
1457	*
1458	* Resets the protocol if needed, gets the new size and adjusts balloon as
1459	* needed. Repeat in 1 sec.
1460	*/
1461	static void vmballoon_work(struct work_struct *work)
1462	{
1463	struct delayed_work *dwork = to_delayed_work(work);
1464	struct vmballoon *b = container_of(dwork, struct vmballoon, dwork);
1465	int64_t change = 0;
1466
1467	if (b->reset_required)
1468	vmballoon_reset(b);
1469
1470	down_read(sem: &b->conf_sem);
1471
1472	/*
1473	* Update the stats while holding the semaphore to ensure that
1474	* @stats_enabled is consistent with whether the stats are actually
1475	* enabled
1476	*/
1477	vmballoon_stats_gen_inc(b, stat: VMW_BALLOON_STAT_TIMER);
1478
1479	if (!vmballoon_send_get_target(b))
1480	change = vmballoon_change(b);
1481
1482	if (change != 0) {
1483	pr_debug("%s - size: %llu, target %lu\n", __func__,
1484	atomic64_read(&b->size), READ_ONCE(b->target));
1485
1486	if (change > 0)
1487	vmballoon_inflate(b);
1488	else /* (change < 0) */
1489	vmballoon_deflate(b, n_frames: 0, coordinated: true);
1490	}
1491
1492	up_read(sem: &b->conf_sem);
1493
1494	/*
1495	* We are using a freezable workqueue so that balloon operations are
1496	* stopped while the system transitions to/from sleep/hibernation.
1497	*/
1498	queue_delayed_work(wq: system_freezable_wq,
1499	dwork, delay: round_jiffies_relative(HZ));
1500
1501	}
1502
1503	/**
1504	* vmballoon_shrinker_scan() - deflate the balloon due to memory pressure.
1505	* @shrinker: pointer to the balloon shrinker.
1506	* @sc: page reclaim information.
1507	*
1508	* Returns: number of pages that were freed during deflation.
1509	*/
1510	static unsigned long vmballoon_shrinker_scan(struct shrinker *shrinker,
1511	struct shrink_control *sc)
1512	{
1513	struct vmballoon *b = &balloon;
1514	unsigned long deflated_frames;
1515
1516	pr_debug("%s - size: %llu", __func__, atomic64_read(&b->size));
1517
1518	vmballoon_stats_gen_inc(b, stat: VMW_BALLOON_STAT_SHRINK);
1519
1520	/*
1521	* If the lock is also contended for read, we cannot easily reclaim and
1522	* we bail out.
1523	*/
1524	if (!down_read_trylock(sem: &b->conf_sem))
1525	return 0;
1526
1527	deflated_frames = vmballoon_deflate(b, n_frames: sc->nr_to_scan, coordinated: true);
1528
1529	vmballoon_stats_gen_add(b, stat: VMW_BALLOON_STAT_SHRINK_FREE,
1530	val: deflated_frames);
1531
1532	/*
1533	* Delay future inflation for some time to mitigate the situations in
1534	* which balloon continuously grows and shrinks. Use WRITE_ONCE() since
1535	* the access is asynchronous.
1536	*/
1537	WRITE_ONCE(b->shrink_timeout, jiffies + HZ * VMBALLOON_SHRINK_DELAY);
1538
1539	up_read(sem: &b->conf_sem);
1540
1541	return deflated_frames;
1542	}
1543
1544	/**
1545	* vmballoon_shrinker_count() - return the number of ballooned pages.
1546	* @shrinker: pointer to the balloon shrinker.
1547	* @sc: page reclaim information.
1548	*
1549	* Returns: number of 4k pages that are allocated for the balloon and can
1550	* therefore be reclaimed under pressure.
1551	*/
1552	static unsigned long vmballoon_shrinker_count(struct shrinker *shrinker,
1553	struct shrink_control *sc)
1554	{
1555	struct vmballoon *b = &balloon;
1556
1557	return atomic64_read(v: &b->size);
1558	}
1559
1560	static void vmballoon_unregister_shrinker(struct vmballoon *b)
1561	{
1562	shrinker_free(shrinker: b->shrinker);
1563	b->shrinker = NULL;
1564	}
1565
1566	static int vmballoon_register_shrinker(struct vmballoon *b)
1567	{
1568	/* Do nothing if the shrinker is not enabled */
1569	if (!vmwballoon_shrinker_enable)
1570	return 0;
1571
1572	b->shrinker = shrinker_alloc(flags: 0, fmt: "vmw-balloon");
1573	if (!b->shrinker)
1574	return -ENOMEM;
1575
1576	b->shrinker->scan_objects = vmballoon_shrinker_scan;
1577	b->shrinker->count_objects = vmballoon_shrinker_count;
1578	b->shrinker->private_data = b;
1579
1580	shrinker_register(shrinker: b->shrinker);
1581
1582	return 0;
1583	}
1584
1585	/*
1586	* DEBUGFS Interface
1587	*/
1588	#ifdef CONFIG_DEBUG_FS
1589
1590	static const char * const vmballoon_stat_page_names[] = {
1591	[VMW_BALLOON_PAGE_STAT_ALLOC] = "alloc",
1592	[VMW_BALLOON_PAGE_STAT_ALLOC_FAIL] = "allocFail",
1593	[VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC] = "errAlloc",
1594	[VMW_BALLOON_PAGE_STAT_REFUSED_FREE] = "errFree",
1595	[VMW_BALLOON_PAGE_STAT_FREE] = "free"
1596	};
1597
1598	static const char * const vmballoon_stat_names[] = {
1599	[VMW_BALLOON_STAT_TIMER] = "timer",
1600	[VMW_BALLOON_STAT_DOORBELL] = "doorbell",
1601	[VMW_BALLOON_STAT_RESET] = "reset",
1602	[VMW_BALLOON_STAT_SHRINK] = "shrink",
1603	[VMW_BALLOON_STAT_SHRINK_FREE] = "shrinkFree"
1604	};
1605
1606	static int vmballoon_enable_stats(struct vmballoon *b)
1607	{
1608	int r = 0;
1609
1610	down_write(sem: &b->conf_sem);
1611
1612	/* did we somehow race with another reader which enabled stats? */
1613	if (b->stats)
1614	goto out;
1615
1616	b->stats = kzalloc(size: sizeof(*b->stats), GFP_KERNEL);
1617
1618	if (!b->stats) {
1619	/* allocation failed */
1620	r = -ENOMEM;
1621	goto out;
1622	}
1623	static_key_enable(key: &balloon_stat_enabled.key);
1624	out:
1625	up_write(sem: &b->conf_sem);
1626	return r;
1627	}
1628
1629	/**
1630	* vmballoon_debug_show - shows statistics of balloon operations.
1631	* @f: pointer to the &struct seq_file.
1632	* @offset: ignored.
1633	*
1634	* Provides the statistics that can be accessed in vmmemctl in the debugfs.
1635	* To avoid the overhead - mainly that of memory - of collecting the statistics,
1636	* we only collect statistics after the first time the counters are read.
1637	*
1638	* Return: zero on success or an error code.
1639	*/
1640	static int vmballoon_debug_show(struct seq_file *f, void *offset)
1641	{
1642	struct vmballoon *b = f->private;
1643	int i, j;
1644
1645	/* enables stats if they are disabled */
1646	if (!b->stats) {
1647	int r = vmballoon_enable_stats(b);
1648
1649	if (r)
1650	return r;
1651	}
1652
1653	/* format capabilities info */
1654	seq_printf(m: f, fmt: "%-22s: %#16x\n", "balloon capabilities",
1655	VMW_BALLOON_CAPABILITIES);
1656	seq_printf(m: f, fmt: "%-22s: %#16lx\n", "used capabilities", b->capabilities);
1657	seq_printf(m: f, fmt: "%-22s: %16s\n", "is resetting",
1658	b->reset_required ? "y" : "n");
1659
1660	/* format size info */
1661	seq_printf(m: f, fmt: "%-22s: %16lu\n", "target", READ_ONCE(b->target));
1662	seq_printf(m: f, fmt: "%-22s: %16llu\n", "current", atomic64_read(v: &b->size));
1663
1664	for (i = 0; i < VMW_BALLOON_CMD_NUM; i++) {
1665	if (vmballoon_cmd_names[i] == NULL)
1666	continue;
1667
1668	seq_printf(m: f, fmt: "%-22s: %16llu (%llu failed)\n",
1669	vmballoon_cmd_names[i],
1670	atomic64_read(v: &b->stats->ops[i][VMW_BALLOON_OP_STAT]),
1671	atomic64_read(v: &b->stats->ops[i][VMW_BALLOON_OP_FAIL_STAT]));
1672	}
1673
1674	for (i = 0; i < VMW_BALLOON_STAT_NUM; i++)
1675	seq_printf(m: f, fmt: "%-22s: %16llu\n",
1676	vmballoon_stat_names[i],
1677	atomic64_read(v: &b->stats->general_stat[i]));
1678
1679	for (i = 0; i < VMW_BALLOON_PAGE_STAT_NUM; i++) {
1680	for (j = 0; j < VMW_BALLOON_NUM_PAGE_SIZES; j++)
1681	seq_printf(m: f, fmt: "%-18s(%s): %16llu\n",
1682	vmballoon_stat_page_names[i],
1683	vmballoon_page_size_names[j],
1684	atomic64_read(v: &b->stats->page_stat[i][j]));
1685	}
1686
1687	return 0;
1688	}
1689
1690	DEFINE_SHOW_ATTRIBUTE(vmballoon_debug);
1691
1692	static void __init vmballoon_debugfs_init(struct vmballoon *b)
1693	{
1694	debugfs_create_file(name: "vmmemctl", S_IRUGO, NULL, data: b,
1695	fops: &vmballoon_debug_fops);
1696	}
1697
1698	static void __exit vmballoon_debugfs_exit(struct vmballoon *b)
1699	{
1700	static_key_disable(key: &balloon_stat_enabled.key);
1701	debugfs_lookup_and_remove(name: "vmmemctl", NULL);
1702	kfree(objp: b->stats);
1703	b->stats = NULL;
1704	}
1705
1706	#else
1707
1708	static inline void vmballoon_debugfs_init(struct vmballoon *b)
1709	{
1710	}
1711
1712	static inline void vmballoon_debugfs_exit(struct vmballoon *b)
1713	{
1714	}
1715
1716	#endif /* CONFIG_DEBUG_FS */
1717
1718
1719	#ifdef CONFIG_BALLOON_COMPACTION
1720	/**
1721	* vmballoon_migratepage() - migrates a balloon page.
1722	* @b_dev_info: balloon device information descriptor.
1723	* @newpage: the page to which @page should be migrated.
1724	* @page: a ballooned page that should be migrated.
1725	* @mode: migration mode, ignored.
1726	*
1727	* This function is really open-coded, but that is according to the interface
1728	* that balloon_compaction provides.
1729	*
1730	* Return: zero on success, -EAGAIN when migration cannot be performed
1731	* momentarily, and -EBUSY if migration failed and should be retried
1732	* with that specific page.
1733	*/
1734	static int vmballoon_migratepage(struct balloon_dev_info *b_dev_info,
1735	struct page *newpage, struct page *page,
1736	enum migrate_mode mode)
1737	{
1738	unsigned long status, flags;
1739	struct vmballoon *b;
1740	int ret;
1741
1742	b = container_of(b_dev_info, struct vmballoon, b_dev_info);
1743
1744	/*
1745	* If the semaphore is taken, there is ongoing configuration change
1746	* (i.e., balloon reset), so try again.
1747	*/
1748	if (!down_read_trylock(sem: &b->conf_sem))
1749	return -EAGAIN;
1750
1751	spin_lock(lock: &b->comm_lock);
1752	/*
1753	* We must start by deflating and not inflating, as otherwise the
1754	* hypervisor may tell us that it has enough memory and the new page is
1755	* not needed. Since the old page is isolated, we cannot use the list
1756	* interface to unlock it, as the LRU field is used for isolation.
1757	* Instead, we use the native interface directly.
1758	*/
1759	vmballoon_add_page(b, idx: 0, p: page);
1760	status = vmballoon_lock_op(b, num_pages: 1, page_size: VMW_BALLOON_4K_PAGE,
1761	op: VMW_BALLOON_DEFLATE);
1762
1763	if (status == VMW_BALLOON_SUCCESS)
1764	status = vmballoon_status_page(b, idx: 0, p: &page);
1765
1766	/*
1767	* If a failure happened, let the migration mechanism know that it
1768	* should not retry.
1769	*/
1770	if (status != VMW_BALLOON_SUCCESS) {
1771	spin_unlock(lock: &b->comm_lock);
1772	ret = -EBUSY;
1773	goto out_unlock;
1774	}
1775
1776	/*
1777	* The page is isolated, so it is safe to delete it without holding
1778	* @pages_lock . We keep holding @comm_lock since we will need it in a
1779	* second.
1780	*/
1781	balloon_page_delete(page);
1782
1783	put_page(page);
1784
1785	/* Inflate */
1786	vmballoon_add_page(b, idx: 0, p: newpage);
1787	status = vmballoon_lock_op(b, num_pages: 1, page_size: VMW_BALLOON_4K_PAGE,
1788	op: VMW_BALLOON_INFLATE);
1789
1790	if (status == VMW_BALLOON_SUCCESS)
1791	status = vmballoon_status_page(b, idx: 0, p: &newpage);
1792
1793	spin_unlock(lock: &b->comm_lock);
1794
1795	if (status != VMW_BALLOON_SUCCESS) {
1796	/*
1797	* A failure happened. While we can deflate the page we just
1798	* inflated, this deflation can also encounter an error. Instead
1799	* we will decrease the size of the balloon to reflect the
1800	* change and report failure.
1801	*/
1802	atomic64_dec(v: &b->size);
1803	ret = -EBUSY;
1804	} else {
1805	/*
1806	* Success. Take a reference for the page, and we will add it to
1807	* the list after acquiring the lock.
1808	*/
1809	get_page(page: newpage);
1810	ret = MIGRATEPAGE_SUCCESS;
1811	}
1812
1813	/* Update the balloon list under the @pages_lock */
1814	spin_lock_irqsave(&b->b_dev_info.pages_lock, flags);
1815
1816	/*
1817	* On inflation success, we already took a reference for the @newpage.
1818	* If we succeed just insert it to the list and update the statistics
1819	* under the lock.
1820	*/
1821	if (ret == MIGRATEPAGE_SUCCESS) {
1822	balloon_page_insert(balloon: &b->b_dev_info, page: newpage);
1823	__count_vm_event(item: BALLOON_MIGRATE);
1824	}
1825
1826	/*
1827	* We deflated successfully, so regardless to the inflation success, we
1828	* need to reduce the number of isolated_pages.
1829	*/
1830	b->b_dev_info.isolated_pages--;
1831	spin_unlock_irqrestore(lock: &b->b_dev_info.pages_lock, flags);
1832
1833	out_unlock:
1834	up_read(sem: &b->conf_sem);
1835	return ret;
1836	}
1837
1838	/**
1839	* vmballoon_compaction_init() - initialized compaction for the balloon.
1840	*
1841	* @b: pointer to the balloon.
1842	*
1843	* If during the initialization a failure occurred, this function does not
1844	* perform cleanup. The caller must call vmballoon_compaction_deinit() in this
1845	* case.
1846	*
1847	* Return: zero on success or error code on failure.
1848	*/
1849	static __init void vmballoon_compaction_init(struct vmballoon *b)
1850	{
1851	b->b_dev_info.migratepage = vmballoon_migratepage;
1852	}
1853
1854	#else /* CONFIG_BALLOON_COMPACTION */
1855	static inline void vmballoon_compaction_init(struct vmballoon *b)
1856	{
1857	}
1858	#endif /* CONFIG_BALLOON_COMPACTION */
1859
1860	static int __init vmballoon_init(void)
1861	{
1862	int error;
1863
1864	/*
1865	* Check if we are running on VMware's hypervisor and bail out
1866	* if we are not.
1867	*/
1868	if (x86_hyper_type != X86_HYPER_VMWARE)
1869	return -ENODEV;
1870
1871	INIT_DELAYED_WORK(&balloon.dwork, vmballoon_work);
1872
1873	error = vmballoon_register_shrinker(b: &balloon);
1874	if (error)
1875	return error;
1876
1877	/*
1878	* Initialization of compaction must be done after the call to
1879	* balloon_devinfo_init() .
1880	*/
1881	balloon_devinfo_init(balloon: &balloon.b_dev_info);
1882	vmballoon_compaction_init(b: &balloon);
1883
1884	INIT_LIST_HEAD(list: &balloon.huge_pages);
1885	spin_lock_init(&balloon.comm_lock);
1886	init_rwsem(&balloon.conf_sem);
1887	balloon.vmci_doorbell = VMCI_INVALID_HANDLE;
1888	balloon.batch_page = NULL;
1889	balloon.page = NULL;
1890	balloon.reset_required = true;
1891
1892	queue_delayed_work(wq: system_freezable_wq, dwork: &balloon.dwork, delay: 0);
1893
1894	vmballoon_debugfs_init(b: &balloon);
1895
1896	return 0;
1897	}
1898
1899	/*
1900	* Using late_initcall() instead of module_init() allows the balloon to use the
1901	* VMCI doorbell even when the balloon is built into the kernel. Otherwise the
1902	* VMCI is probed only after the balloon is initialized. If the balloon is used
1903	* as a module, late_initcall() is equivalent to module_init().
1904	*/
1905	late_initcall(vmballoon_init);
1906
1907	static void __exit vmballoon_exit(void)
1908	{
1909	vmballoon_unregister_shrinker(b: &balloon);
1910	vmballoon_vmci_cleanup(b: &balloon);
1911	cancel_delayed_work_sync(dwork: &balloon.dwork);
1912
1913	vmballoon_debugfs_exit(b: &balloon);
1914
1915	/*
1916	* Deallocate all reserved memory, and reset connection with monitor.
1917	* Reset connection before deallocating memory to avoid potential for
1918	* additional spurious resets from guest touching deallocated pages.
1919	*/
1920	vmballoon_send_start(b: &balloon, req_caps: 0);
1921	vmballoon_pop(b: &balloon);
1922	}
1923	module_exit(vmballoon_exit);
1924